diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rs index b328583b6..0e565d795 100644 --- a/crates/utopia-server/src/extraction_open.rs +++ b/crates/utopia-server/src/extraction_open.rs @@ -80,6 +80,56 @@ fn mention_evidence( evidence } +/// 补漏不能把不同引文、同句多个主体或重复出现的名字压成一个身份。 +/// 这里只接受唯一定位的一句,并要求该主体的每条陈述都引用它。 +fn omitted_subject_quote<'a>( + extraction: &'a utopia_extract::open::OpenExtraction, + text: &str, + key: &str, +) -> Option<&'a str> { + if extraction + .names + .iter() + .any(|n| name_key(&n.entity) == key || name_key(&n.name) == key) + { + return None; + } + let mut quotes = extraction + .statements + .iter() + .filter(|s| name_key(&s.subject) == key) + .map(|s| s.quote.as_deref().map(str::trim)); + let quote = quotes.next()??; + let (start, end) = locate(text, quote)?; + let body = quote.trim_end_matches(['.', '!', '?', '。', '!', '?', '"', '”', '’']); + if !span_in_quote(key, quote) + || body.contains(['.', '!', '?', '。', '!', '?', ';', ';']) + || quotes.any(|other| other != Some(quote)) + || extraction.statements.iter().any(|other| { + name_key(&other.subject) != key + && other + .quote + .as_deref() + .and_then(|q| locate(text, q)) + .is_some_and(|(a, b)| a < end && start < b) + }) + || extraction.entities.iter().any(|e| { + let declared = name_key(&e.name); + declared != key && declared.contains(key) && span_in_quote(&e.name, quote) + }) + { + return None; + } + let once = |hay: &[char], needle: &[char]| { + !needle.is_empty() && hay.windows(needle.len()).filter(|w| *w == needle).count() == 1 + }; + let (body, _) = squashed(text); + let (sentence, _) = squashed(quote); + let subject: Vec<_> = key.chars().collect(); + let normalized_quote: Vec<_> = name_key(quote).chars().collect(); + (once(&body, &sentence) && once(&normalized_quote, &subject)).then_some(quote) +} + /// 回复为什么是半截的:端点撞上了 token 上限(#760),或护栏看见它在复读、停了读(#1301)。 /// 两种都当半截处理,丢弃行要说清是哪一种——复读的去查上限,是找错了地方 #[derive(Debug, Clone, Copy)] @@ -373,23 +423,65 @@ async fn apply_open_extraction( local.insert(key, id); } + // 漏列的主体只能查回唯一的既有名字,不能借补漏创建或重新分类实体。 + // 另存映射只供主体使用,避免顺带改变未声明宾语的字面值回退(#559)。 + let mut recovered_subjects = HashMap::new(); + if !pushed && !identity && !await_nod { + let declared: HashSet<_> = extraction + .entities + .iter() + .map(|e| name_key(&e.name)) + .collect(); + let subjects: HashSet<_> = extraction + .statements + .iter() + .map(|s| name_key(&s.subject)) + .collect(); + for key in &subjects { + if declared.contains(key) + || known_by_name.contains_key(key) + || handled_by_name.contains_key(key) + { + continue; + } + if omitted_subject_quote(extraction, &chunk.text, key).is_some() { + if let Some(id) = utopia_store::resolution::recover_existing_subject( + pool, + kb_id, + key, + ctx, + evidence.get(key).map(String::as_str), + ) + .await? + { + recovered_subjects.insert(key.clone(), id); + } + } + } + } + // ---- 陈述 ---- for s in &extraction.statements { let phrase = s.phrase.trim(); if phrase.is_empty() { continue; } - let Some(subject) = place( - pool, - kb_id, - &mut local, - known_by_name, - &deferred, - described, - &s.subject, - ) - .await? - else { + let subject = match recovered_subjects.get(&name_key(&s.subject)) { + Some(id) => Some(*id), + None => { + place( + pool, + kb_id, + &mut local, + known_by_name, + &deferred, + described, + &s.subject, + ) + .await? + } + }; + let Some(subject) = subject else { drop_signal( state, kb_id, diff --git a/crates/utopia-server/src/namesake_missing_subject_cases.rs b/crates/utopia-server/src/namesake_missing_subject_cases.rs new file mode 100644 index 000000000..38e052660 --- /dev/null +++ b/crates/utopia-server/src/namesake_missing_subject_cases.rs @@ -0,0 +1,601 @@ +//! 漏列主体只允许恢复唯一既有名字;不修改原来的输入、金标准或具名失败列表。 +use super::*; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Missing { + Canonical, + ExistingAlias, + Typed, + MergedOther, + Ambiguous, + Absent, + Forged, + NoQuote, + CrossSentence, + SharedQuote, + RepeatedQuote, + RepeatedName, + MultipleQuotes, + Described, + AliasDeclaration, + CompoundName, + ObjectOnly, + Pushed, + Memory, + DeclaredControl, + ForeignBase, + PdfWhitespace, + Chinese, + LowProfile, + GreyProfile, + NoProfile, + TypedNoProfile, + MixedForgedQuote, + SubjectAndObject, + DeclaredLongAlias, + Bare, + GreyNoClue, + OtherSubjectHigh, + OtherSubjectGrey, + InvalidatedAlias, + DescribedHolder, + GenericCompetitor, +} + +async fn run_missing(fx: &Fx, case: Missing, reverse: bool) -> anyhow::Result { + let mut holder = None; + let mut retired = None; + for i in if reverse { [1, 0] } else { [0, 1] } { + if i == 0 && case != Missing::Absent { + holder = Some(fx.seed(0, Check::Resolver { target: 0 }).await?); + } else if i == 1 && matches!(case, Missing::Ambiguous | Missing::MergedOther) { + let other = fx.seed(1, Check::Resolver { target: 0 }).await?; + if case == Missing::MergedOther { + // 回收已经合并的记录不能让一个活持有者变成两个。 + retired = Some(other); + } + } else { + sqlx::query( + "INSERT INTO entities(id,kb_id,canonical_name) VALUES ($1,$2,'Other Person')", + ) + .bind(Uuid::now_v7()) + .bind(fx.kb) + .execute(&fx.pool) + .await?; + } + } + if case == Missing::MergedOther { + sqlx::query("UPDATE entities SET merged_into=$1 WHERE id=$2") + .bind(holder) + .bind(retired) + .execute(&fx.pool) + .await?; + } + if matches!(case, Missing::ExistingAlias | Missing::InvalidatedAlias) { + sqlx::query("UPDATE entities SET canonical_name='Alexander Lee' WHERE id=$1") + .bind(holder) + .execute(&fx.pool) + .await?; + utopia_store::names::record(&fx.pool, fx.kb, holder.unwrap(), "Alex Lee", None, None) + .await?; + } + if case == Missing::InvalidatedAlias { + sqlx::query("UPDATE facts SET invalidated_at=now() WHERE kb_id=$1 AND subject_id=$2 AND object_value->>'value'='Alex Lee'").bind(fx.kb).bind(holder).execute(&fx.pool).await?; + } + if case == Missing::DescribedHolder { + sqlx::query("UPDATE entities SET description='a similarly described thing' WHERE id=$1") + .bind(holder) + .execute(&fx.pool) + .await?; + } + if case == Missing::GenericCompetitor { + sqlx::query("UPDATE entities SET canonical_name='Alex Lee Team' WHERE kb_id=$1 AND canonical_name='Other Person'").bind(fx.kb).execute(&fx.pool).await?; + } + if matches!(case, Missing::Typed | Missing::TypedNoProfile) { + let class = Uuid::now_v7(); + sqlx::query( + "INSERT INTO entity_types(id,kb_id,key,label) VALUES ($1,$2,'person','Person')", + ) + .bind(class) + .bind(fx.kb) + .execute(&fx.pool) + .await?; + sqlx::query("UPDATE entities SET type_id=$1,type_source='human' WHERE id=$2") + .bind(class) + .bind(holder) + .execute(&fx.pool) + .await?; + } + if matches!(case, Missing::NoProfile | Missing::TypedNoProfile) { + sqlx::query("UPDATE entities SET profile_embedding=NULL WHERE id=$1") + .bind(holder) + .execute(&fx.pool) + .await?; + } + if matches!( + case, + Missing::LowProfile + | Missing::GreyProfile + | Missing::GreyNoClue + | Missing::OtherSubjectGrey + ) { + let vector = if case == Missing::LowProfile { + "[0,1,0]" + } else { + "[0.5,0.866,0]" + }; + sqlx::query("UPDATE entities SET profile_embedding=$1::text::vector WHERE id=$2") + .bind(vector) + .bind(holder) + .execute(&fx.pool) + .await?; + } + if case == Missing::ForeignBase { + let ws = Uuid::now_v7(); + let kb = Uuid::now_v7(); + sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES ($1,$2,'other scope')") + .bind(ws) + .bind(fx.org) + .execute(&fx.pool) + .await?; + sqlx::query( + "INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ($1,$2,'other scope')", + ) + .bind(kb) + .bind(ws) + .execute(&fx.pool) + .await?; + sqlx::query("INSERT INTO entities(id,kb_id,canonical_name) VALUES ($1,$2,'Alex Lee')") + .bind(Uuid::now_v7()) + .bind(kb) + .execute(&fx.pool) + .await?; + } + let initial_type: Option<(Option, Option, String)> = match holder { + Some(id) => Some( + sqlx::query_as("SELECT type_id,specific_type,type_source FROM entities WHERE id=$1") + .bind(id) + .fetch_one(&fx.pool) + .await?, + ), + None => None, + }; + let ordinary = "Alex Lee from Platform Engineering submitted a report."; + let mut text = ordinary.to_string(); + let mut quote = Some(ordinary.to_string()); + let mut subject = "Alex Lee"; + let mut entities = json!([]); + let mut names = json!([]); + let mut object = Value::Null; + let mut value = json!("report"); + match case { + Missing::Bare | Missing::GreyNoClue => { + text = "Alex Lee submitted a report.".into(); + quote = Some(text.clone()); + } + Missing::Forged => quote = Some("Alex Lee won a prize.".into()), + Missing::NoQuote => quote = None, + Missing::CrossSentence => { + text = "Alex Lee spoke. Platform Engineering submitted a report.".into(); + quote = Some(text.clone()); + } + Missing::RepeatedQuote => text = format!("{ordinary} {ordinary}"), + Missing::RepeatedName => { + text = "Alex Lee thanked Alex Lee for a report.".into(); + quote = Some(text.clone()); + } + Missing::Described => entities = json!([["Alex Lee", "role", 0]]), + Missing::AliasDeclaration => { + text = "Alexander Lee, also known as Alex Lee, submitted a report.".into(); + quote = Some(text.clone()); + entities = json!([["Alexander Lee", "person", 1]]); + names = json!([["Alexander Lee", "Alex Lee", text]]); + } + Missing::CompoundName => { + subject = "上海"; + text = "上海市政府发布报告。".into(); + quote = Some(text.clone()); + sqlx::query("UPDATE entities SET canonical_name='上海' WHERE id=$1") + .bind(holder) + .execute(&fx.pool) + .await?; + entities = json!([["上海市政府", "organization", 1]]); + } + Missing::DeclaredLongAlias => { + entities = json!([["Alex Lee Society", "organization", 1]]); + names = json!([["Alex Lee Society", "Alex Lee", ordinary]]); + } + Missing::SubjectAndObject => { + entities = json!([["Caller", "organization", 1]]); + } + Missing::ObjectOnly => { + text = "Caller assigned a report to Alex Lee.".into(); + quote = Some(text.clone()); + subject = "Caller"; + entities = json!([["Caller", "organization", 1]]); + object = json!("Alex Lee"); + value = Value::Null; + } + Missing::DeclaredControl => entities = json!([["Alex Lee", "person", 1]]), + Missing::PdfWhitespace => { + text = "Alex\n Lee from Platform Engineering submitted a report.".into() + } + Missing::Chinese => { + subject = "李娜"; + text = "李娜提交报告。".into(); + quote = Some(text.clone()); + sqlx::query("UPDATE entities SET canonical_name='李娜' WHERE id=$1") + .bind(holder) + .execute(&fx.pool) + .await?; + } + _ => {} + } + let mut statements = vec![json!([ + quote, + subject, + "submitted", + object, + value, + null, + null, + null + ])]; + if case == Missing::SharedQuote { + text = "Alex Lee and Bob Jones submitted a report.".into(); + statements = vec![ + json!([ + text, + "Alex Lee", + "submitted", + null, + "report", + null, + null, + null + ]), + json!([ + text, + "Bob Jones", + "submitted", + null, + "report", + null, + null, + null + ]), + ]; + } + if case == Missing::MultipleQuotes { + let second = "Alex Lee from Finance submitted an annual report."; + text = format!("{ordinary} {second}"); + statements.push(json!([ + second, + "Alex Lee", + "submitted", + null, + "annual report", + null, + null, + null + ])); + } + if matches!(case, Missing::OtherSubjectHigh | Missing::OtherSubjectGrey) { + text = format!("{ordinary} Platform Engineering filed a budget."); + entities = json!([["Platform Engineering", "department", 1]]); + statements.push(json!([ + "Platform Engineering filed a budget.", + "Platform Engineering", + "filed", + null, + "budget", + null, + null, + null + ])); + } + if case == Missing::MixedForgedQuote { + statements.push(json!([ + "Alex Lee won a prize.", + "Alex Lee", + "submitted", + null, + "prize", + null, + null, + null + ])); + } + if case == Missing::SubjectAndObject { + text = format!("{ordinary} Caller assigned a report to Alex Lee."); + statements.push(json!([ + "Caller assigned a report to Alex Lee.", + "Caller", + "submitted", + "Alex Lee", + null, + null, + null, + null + ])); + } + let reply = json!({"e":entities,"s":statements,"n":names}).to_string(); + if matches!(case, Missing::Pushed | Missing::Memory) { + let source = Uuid::now_v7(); + let kind = if case == Missing::Pushed { + "statements" + } else { + "memory" + }; + sqlx::query( + "INSERT INTO sources(id,kb_id,kind,name,config) VALUES ($1,$2,$3,'guard','{}')", + ) + .bind(source) + .bind(fx.kb) + .bind(kind) + .execute(&fx.pool) + .await?; + sqlx::query("UPDATE documents SET source_id=$1 WHERE id=$2") + .bind(source) + .bind(fx.docs[2]) + .execute(&fx.pool) + .await?; + if case == Missing::Pushed { + text = reply.clone(); + } + } + sqlx::query("UPDATE chunks SET text=$1 WHERE id=$2") + .bind(&text) + .bind(fx.chunks[2]) + .execute(&fx.pool) + .await?; + if case != Missing::Pushed { + fx.model.steps.lock().unwrap().push_back(Step { + prompt_has: if case == Missing::PdfWhitespace { + "Alex".into() + } else { + text.clone() + }, + stream: true, + text: reply, + }); + } + let entities_before: i64 = sqlx::query_scalar("SELECT count(*) FROM entities WHERE kb_id=$1") + .bind(fx.kb) + .fetch_one(&fx.pool) + .await?; + crate::extraction::extract_document(&fx.state, fx.docs[2], Default::default()).await?; + let facts:Vec<(Uuid,Option,Option)>=sqlx::query_as( + "SELECT f.subject_id,f.object_id,f.object_value FROM facts f JOIN fact_evidence fe ON fe.fact_id=f.id WHERE f.kb_id=$1 AND fe.document_id=$2 AND f.layer='open' AND f.phrase='submitted' ORDER BY f.id") + .bind(fx.kb).bind(fx.docs[2]).fetch_all(&fx.pool).await?; + let recovered = matches!( + case, + Missing::Canonical + | Missing::ExistingAlias + | Missing::Typed + | Missing::MergedOther + | Missing::DeclaredControl + | Missing::ForeignBase + | Missing::PdfWhitespace + | Missing::Chinese + | Missing::GreyProfile + | Missing::NoProfile + | Missing::Bare + | Missing::OtherSubjectHigh + ); + let correct = if recovered { + facts.len() == 1 && Some(facts[0].0) == holder + } else if case == Missing::SubjectAndObject { + facts.len() == 2 + && facts.iter().any(|f| Some(f.0) == holder) + && facts.iter().any(|f| { + Some(f.0) != holder + && f.1.is_none() + && f.2.as_ref().is_some_and(|v| v["value"] == "Alex Lee") + }) + } else if case == Missing::Described { + facts.len() == 1 + && Some(facts[0].0) != holder + && sqlx::query_scalar::<_, bool>( + "SELECT description IS NOT NULL FROM entities WHERE id=$1", + ) + .bind(facts[0].0) + .fetch_one(&fx.pool) + .await? + } else if case == Missing::ObjectOnly { + facts.len() == 1 + && facts[0].1.is_none() + && facts[0] + .2 + .as_ref() + .is_some_and(|v| v["value"] == "Alex Lee") + } else { + facts.is_empty() + }; + let pending: i64 = sqlx::query_scalar( + "SELECT count(*) FROM resolution_reviews WHERE kb_id=$1 AND status='pending'", + ) + .bind(fx.kb) + .fetch_one(&fx.pool) + .await?; + let entities_after: i64 = sqlx::query_scalar("SELECT count(*) FROM entities WHERE kb_id=$1") + .bind(fx.kb) + .fetch_one(&fx.pool) + .await?; + let type_unchanged = if matches!(case, Missing::Typed | Missing::TypedNoProfile) { + initial_type + == Some( + sqlx::query_as( + "SELECT type_id,specific_type,type_source FROM entities WHERE id=$1", + ) + .bind(holder) + .fetch_one(&fx.pool) + .await?, + ) + } else { + true + }; + // 再次抽取已完成的块不能新增事实、实体或模型请求。 + let calls = fx.model.requests.lock().unwrap().len(); + crate::extraction::extract_document(&fx.state, fx.docs[2], Default::default()).await?; + let final_entities: i64 = sqlx::query_scalar("SELECT count(*) FROM entities WHERE kb_id=$1") + .bind(fx.kb) + .fetch_one(&fx.pool) + .await?; + let final_facts:i64=sqlx::query_scalar("SELECT count(*) FROM facts f JOIN fact_evidence fe ON fe.fact_id=f.id WHERE f.kb_id=$1 AND fe.document_id=$2 AND f.layer='open' AND f.phrase='submitted'").bind(fx.kb).bind(fx.docs[2]).fetch_one(&fx.pool).await?; + let final_pending: i64 = sqlx::query_scalar( + "SELECT count(*) FROM resolution_reviews WHERE kb_id=$1 AND status='pending'", + ) + .bind(fx.kb) + .fetch_one(&fx.pool) + .await?; + let repeated_calls = fx.model.requests.lock().unwrap().len(); + let no_extra_entities = matches!( + case, + Missing::Described + | Missing::AliasDeclaration + | Missing::CompoundName + | Missing::ObjectOnly + | Missing::SubjectAndObject + | Missing::DeclaredLongAlias + ) || entities_before == entities_after; + Ok( + json!({"case":format!("{case:?}"),"reverse":reverse,"passed":correct&&pending==0&&type_unchanged&&no_extra_entities&&final_entities==entities_after&&calls==repeated_calls&&final_facts==facts.len() as i64&&final_pending==pending, + "facts":facts,"existing_holder":holder,"entities_before":entities_before,"entities_after":entities_after, + "pending_reviews":pending,"type_unchanged":type_unchanged,"model_calls":calls,"repeated_model_calls":repeated_calls}), + ) +} + +#[tokio::test] +async fn missing_subjects_only_recover_grounded_unique_existing_names() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = utopia_store::db::connect(&url, Some(4)).await?; + let cases = [ + Missing::Canonical, + Missing::ExistingAlias, + Missing::Typed, + Missing::MergedOther, + Missing::Ambiguous, + Missing::Absent, + Missing::Forged, + Missing::NoQuote, + Missing::CrossSentence, + Missing::SharedQuote, + Missing::RepeatedQuote, + Missing::RepeatedName, + Missing::MultipleQuotes, + Missing::Described, + Missing::AliasDeclaration, + Missing::CompoundName, + Missing::ObjectOnly, + Missing::Pushed, + Missing::Memory, + Missing::DeclaredControl, + Missing::ForeignBase, + Missing::PdfWhitespace, + Missing::Chinese, + Missing::LowProfile, + Missing::GreyProfile, + Missing::NoProfile, + Missing::TypedNoProfile, + Missing::MixedForgedQuote, + Missing::SubjectAndObject, + Missing::DeclaredLongAlias, + Missing::Bare, + Missing::GreyNoClue, + Missing::OtherSubjectHigh, + Missing::OtherSubjectGrey, + Missing::InvalidatedAlias, + Missing::DescribedHolder, + Missing::GenericCompetitor, + ]; + let mut failures = Vec::new(); + for case in cases { + for reverse in [false, true] { + let template = Case { + id: "missing-subject", + check: Check::Resolver { target: 0 }, + }; + let fx = Fx::new(&pool, &template, Uuid::now_v7()).await?; + let result = run_missing(&fx, case, reverse).await; + let cleanup = fx.cleanup().await; + let observation = result?; + cleanup?; + println!("GROUNDED_SUBJECT_CASE {observation}"); + if observation["passed"] != true { + failures.push(observation) + } + } + } + anyhow::ensure!( + failures.is_empty(), + "subject recovery mismatches: {}", + json!(failures) + ); + pool.close().await; + Ok(()) +} + +/// 保留三份历史回复与原文;这里只重放抽取落库,不冒充新模型测量或完整历史图重放。 +/// 画像用等余弦的三维向量隔离门槛,完整 BGE-M3 向量仍留在冻结快照中。 +#[tokio::test] +async fn historical_omitted_subject_replies_keep_grounded_facts() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = utopia_store::db::connect(&url, Some(4)).await?; + let rows: Vec = serde_json::from_str(include_str!( + "../tests/fixtures/grounded_subject_replies.json" + ))?; + let mut failures = Vec::new(); + for row in rows { + for reverse in [false, true] { + let template = Case { + id: "historical-missing-subject", + check: Check::Resolver { target: 0 }, + }; + let fx = Fx::new(&pool, &template, Uuid::now_v7()).await?; + let result=async { + let mut holder=None; + for i in if reverse {[1,0]} else {[0,1]} { + if i==0 && row["has_holder"]==true { + let id=fx.seed(0,Check::Resolver {target:0}).await?; + let class=Uuid::now_v7(); + sqlx::query("INSERT INTO entity_types(id,kb_id,key,label) VALUES ($1,$2,'person','Person')").bind(class).bind(fx.kb).execute(&pool).await?; + let score=row["profile_cosine"].as_f64().unwrap(); + let profile=format!("[{score},{},0]",(1.0-score*score).sqrt()); + sqlx::query("UPDATE entities SET canonical_name='李娜',type_id=$1,type_source='human',profile_embedding=$2::text::vector WHERE id=$3").bind(class).bind(profile).bind(id).execute(&pool).await?; + holder=Some(id); + } else { + sqlx::query("INSERT INTO entities(id,kb_id,canonical_name) VALUES ($1,$2,'Other Person')").bind(Uuid::now_v7()).bind(fx.kb).execute(&pool).await?; + } + } + let text=row["text"].as_str().unwrap(); + sqlx::query("UPDATE chunks SET text=$1 WHERE id=$2").bind(text).bind(fx.chunks[2]).execute(&pool).await?; + fx.model.steps.lock().unwrap().push_back(Step {prompt_has:"李娜主持编写".into(),stream:true,text:row["reply"].as_str().unwrap().into()}); + crate::extraction::extract_document(&fx.state,fx.docs[2],Default::default()).await?; + let subjects:Vec=sqlx::query_scalar("SELECT f.subject_id FROM facts f JOIN fact_evidence fe ON fe.fact_id=f.id WHERE f.kb_id=$1 AND fe.document_id=$2 AND f.layer='open' AND f.phrase='主持编写了'").bind(fx.kb).bind(fx.docs[2]).fetch_all(&pool).await?; + let people:i64=sqlx::query_scalar("SELECT count(*) FROM entities WHERE kb_id=$1 AND canonical_name='李娜' AND merged_into IS NULL").bind(fx.kb).fetch_one(&pool).await?; + let pending:i64=sqlx::query_scalar("SELECT count(*) FROM resolution_reviews WHERE kb_id=$1 AND status='pending'").bind(fx.kb).fetch_one(&pool).await?; + let passed=match holder {Some(id)=>subjects==vec![id] && people==1,None=>subjects.is_empty() && people==0}; + Ok::<_,anyhow::Error>(json!({"snapshot":row["snapshot"],"reverse_seed":reverse,"subjects":subjects,"people":people,"pending_reviews":pending,"model_calls":fx.model.requests.lock().unwrap().len(),"passed":passed&&pending==0})) + }.await; + let cleanup = fx.cleanup().await; + let observation = result?; + cleanup?; + println!("HISTORICAL_SUBJECT_CASE {observation}"); + if observation["passed"] != true { + failures.push(observation); + } + } + } + anyhow::ensure!( + failures.is_empty(), + "historical subject recovery mismatches: {}", + json!(failures) + ); + pool.close().await; + Ok(()) +} diff --git a/crates/utopia-server/src/namesake_scripted_cases.rs b/crates/utopia-server/src/namesake_scripted_cases.rs index b5be0f345..fcf48521c 100644 --- a/crates/utopia-server/src/namesake_scripted_cases.rs +++ b/crates/utopia-server/src/namesake_scripted_cases.rs @@ -181,6 +181,9 @@ async fn reply(State(m): State, Json(body): Json) -> axum::respons .into_response() } +#[path = "namesake_missing_subject_cases.rs"] +mod missing_subject_cases; + struct Fx { pool: sqlx::PgPool, state: AppState, diff --git a/crates/utopia-server/tests/fixtures/grounded_subject_replies.json b/crates/utopia-server/tests/fixtures/grounded_subject_replies.json new file mode 100644 index 000000000..160c36382 --- /dev/null +++ b/crates/utopia-server/tests/fixtures/grounded_subject_replies.json @@ -0,0 +1,23 @@ +[ + { + "snapshot": "request-states/candidate-0165.json.gz", + "has_holder": true, + "profile_cosine": 0.6764983611798704, + "text": "海洋探测器2号耐压舱规格(2023年10月12日)\n\n李娜主持编写了海洋探测器2号(简称“海探2”)的耐压舱技术规格。海探2的设计深度为11000米。", + "reply": "{\n\"e\": [\n[\"海洋探测器2号\", \"product\", 1],\n[\"海探2\", \"abbreviation\", 1],\n[\"耐压舱技术规格\", \"document\", 0]\n],\n\"s\": [\n[\"李娜主持编写了海洋探测器2号(简称“海探2”)的耐压舱技术规格。\", \"李娜\", \"主持编写了\", \"耐压舱技术规格\", null, {\"for\": \"海洋探测器2号\"}, \"2023年10月12日\", null],\n[\"海探2的设计深度为11000米。\", \"海探2\", \"设计深度为\", null, \"11000米\", null, null, null]\n],\n\"n\": [\n[\"海洋探测器2号\", \"海探2\", \"海洋探测器2号(简称“海探2”)\"]\n],\n\"t\": []\n}" + }, + { + "snapshot": "request-states/candidate-0295.json.gz", + "has_holder": true, + "profile_cosine": 0.6764983611798704, + "text": "海洋探测器2号耐压舱规格(2023年10月12日)\n\n李娜主持编写了海洋探测器2号(简称“海探2”)的耐压舱技术规格。海探2的设计深度为11000米。", + "reply": "{\n\"e\": [\n[\"海洋探测器2号\", \"product\", 1],\n[\"海探2\", \"product\", 0],\n[\"耐压舱\", \"component\", 0]\n],\n\"s\": [\n[\"李娜主持编写了海洋探测器2号(简称“海探2”)的耐压舱技术规格。\", \"李娜\", \"主持编写了\", \"耐压舱技术规格\", null, {\"for\": \"海洋探测器2号\"}, null, null],\n[\"海探2的设计深度为11000米。\", \"海探2\", \"设计深度为\", null, \"11000米\", null, null, null]\n],\n\"n\": [\n[\"海洋探测器2号\", \"海探2\", \"海洋探测器2号(简称“海探2”)\"]\n],\n\"t\": []\n}" + }, + { + "snapshot": "request-states/candidate-0347.json.gz", + "has_holder": false, + "profile_cosine": null, + "text": "海洋探测器2号耐压舱规格(2023年10月12日)\n\n李娜主持编写了海洋探测器2号(简称“海探2”)的耐压舱技术规格。海探2的设计深度为11000米。", + "reply": "{\n\"e\": [\n[\"海洋探测器2号\", \"product\", 1],\n[\"耐压舱技术规格\", \"document\", 0],\n[\"海探2\", \"product\", 1]\n],\n\"s\": [\n[\"李娜主持编写了海洋探测器2号(简称“海探2”)的耐压舱技术规格。\", \"李娜\", \"主持编写了\", \"耐压舱技术规格\", null, {\"for\": \"海洋探测器2号\"}, null, null],\n[\"海探2的设计深度为11000米。\", \"海探2\", \"设计深度为\", null, \"11000米\", null, null, null]\n],\n\"n\": [\n[\"海洋探测器2号\", \"海探2\", \"海洋探测器2号(简称“海探2”)\"]\n],\n\"t\": []\n}" + } +] diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rs index b7f34250a..5c3844aef 100644 --- a/crates/utopia-store/src/resolution.rs +++ b/crates/utopia-store/src/resolution.rs @@ -98,6 +98,65 @@ pub fn name_key(raw: &str) -> String { fold_markdown_markers(&normalize_name(raw)).to_lowercase() } +/// 模型漏列主体时只补回已有的、能按原画像规则归属的名字;不能借补漏创建实体, +/// 也不能把「只有一个名字」当作身份声明(0071 的直接归并只属于身份来源)。 +pub async fn recover_existing_subject( + pool: &PgPool, + kb_id: Uuid, + raw_name: &str, + context: Option<&[f32]>, + text: Option<&str>, +) -> AppResult> { + let normalized = normalize_name(raw_name); + let exact = vec![normalized.to_lowercase(), name_key(raw_name)]; + let keys = recall_keys(&normalized); + let holders: Vec = sqlx::query_as(&format!( + "SELECT e.id, e.canonical_name, e.type_id, e.profile_embedding, e.profile_n, + (lower(e.canonical_name) = ANY($3) OR {exact_named}) AS exact_name + FROM entities e + WHERE e.kb_id = $1 AND e.merged_into IS NULL AND e.description IS NULL + AND (lower(e.canonical_name) = ANY($2) OR {named}) + LIMIT 2", + named = crate::names::has_name_in("e", 1, 2), + exact_named = crate::names::has_name_in("e", 1, 3), + )) + .bind(kb_id) + .bind(&keys) + .bind(&exact) + .fetch_all(pool) + .await?; + let [holder] = holders.as_slice() else { + return Ok(None); + }; + if !holder.exact_name { + return Ok(None); + } + let score = context.and_then(|ctx| { + holder + .profile_embedding + .as_ref() + .and_then(|profile| cosine(profile.as_slice(), ctx)) + }); + // 漏列意味着没有类型声明。同类型的无画像兼容及灰区旁证沿用原规则; + // 已分类候选走类型漂移的高分门槛,不能用无类型补漏来降低它。 + let attach = if holder.type_id.is_some() { + score.is_some_and(|score| score >= SIM_ATTACH) + } else { + match score { + None => true, + Some(score) if score >= SIM_ATTACH => true, + Some(score) if score >= SIM_NEW => match text { + Some(text) => corroborating_candidate(pool, kb_id, &[holder], text) + .await? + .is_some(), + None => false, + }, + Some(_) => false, + } + }; + Ok(attach.then_some(holder.id)) +} + /// 泛用后缀词表:中文直接拼在词干后;英文按独立词算、首尾两种语序都认 /// ("Phoenix Project" / "Project Phoenix")。只影响召回,判定仍走画像相似度。 const GENERIC_SUFFIXES_CJK: &[&str] = &["项目", "公司", "集团", "部门", "团队"];