diff --git a/crates/toolbox/src/fixture.rs b/crates/toolbox/src/fixture.rs index 77967ee9..82e90597 100644 --- a/crates/toolbox/src/fixture.rs +++ b/crates/toolbox/src/fixture.rs @@ -88,6 +88,10 @@ pub struct RecordedPage { pub title: Option, #[serde(default)] pub delay_ms: u64, + /// Where the read ended after redirects (a Google News link's publisher + /// page); absent means the backend did not say. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub final_url: Option, /// Replay a failed read (blocked, no main text, …). #[serde(default)] pub error: Option, @@ -195,6 +199,7 @@ impl ResearchBackend for FixtureBackend { Ok(PageText { text: page.text.clone(), title: page.title.clone(), + final_url: page.final_url.clone(), }) }) } diff --git a/crates/toolbox/src/host.rs b/crates/toolbox/src/host.rs index a7657a42..2cc6016a 100644 --- a/crates/toolbox/src/host.rs +++ b/crates/toolbox/src/host.rs @@ -139,6 +139,15 @@ pub struct Provenance { /// SHA-256 of the evidence text the host kept, once read. #[serde(default, skip_serializing_if = "Option::is_none")] pub evidence_sha256: Option, + /// Where the read of it ended, after redirects (a Google News link's + /// publisher page); absent when not read or still on the aggregator. + /// `url` stays the search result's link. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resolved_url: Option, + /// The publisher's host (lowercase, no `www.`): `resolved_url`'s, else + /// `url`'s unless that is an aggregator's. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub publisher_host: Option, /// The provider or method that produced it (`search:gdelt`, `article`). pub via: String, } diff --git a/crates/toolbox/src/library.rs b/crates/toolbox/src/library.rs index 762fe59f..89c92f45 100644 --- a/crates/toolbox/src/library.rs +++ b/crates/toolbox/src/library.rs @@ -146,6 +146,7 @@ pub static BUILTIN: &[(&str, &str, &str)] = builtin!( "market-brief", "briefing", "compare", + "dossier", ); const BUILTIN_LOCK: &str = include_str!("../templates/library.lock.json"); diff --git a/crates/toolbox/src/modules.rs b/crates/toolbox/src/modules.rs index 1c73d034..2e3acca4 100644 --- a/crates/toolbox/src/modules.rs +++ b/crates/toolbox/src/modules.rs @@ -111,7 +111,7 @@ pub const RESEARCH: ModuleSpec = ModuleSpec { MethodSpec { name: "search", kind: MethodKind::Search, - description: "One query through the provider chain; structured items with host-assigned ids, readable items first (`readable: false` marks one the backend cannot read)", + description: "One query through the provider chain (the metasearch `category`, `news` by default, must be in the app's grant; `max_age_hours` is clamped to its `max_age_days`); structured items with host-assigned ids, readable items first (`readable: false` marks one the backend cannot read)", input: || { json!({"type": "object", "additionalProperties": false, "required": ["topic"], "properties": { @@ -119,7 +119,8 @@ pub const RESEARCH: ModuleSpec = ModuleSpec { "language": string(2, 16), "region": string(2, 8), "limit": {"type": "integer", "minimum": 1, "maximum": 10}, - "max_age_hours": {"type": "integer", "minimum": 1, "maximum": 720} + "max_age_hours": {"type": "integer", "minimum": 1, "maximum": 36000}, + "category": {"type": "string", "enum": ["news", "general", "it", "social", "science"]} }}) }, output: || { @@ -141,19 +142,21 @@ pub const RESEARCH: ModuleSpec = ModuleSpec { MethodSpec { name: "article", kind: MethodKind::Read, - description: "Read one search result's page and keep its main text as evidence; `on_topic: false` when the page does not mention the terms of the search that found it (the host will not digest it)", + description: "Read one search result's page and keep its main text as evidence; `resolved_url` is where the read ended (a Google News link's publisher page, null while unresolved) and `publisher_host` its host; `on_topic: false` when the page does not mention the terms of the search that found it (the host will not digest it)", input: || { json!({"type": "object", "additionalProperties": false, "required": ["id"], "properties": {"id": string(1, 64)}}) }, output: || { json!({"type": "object", "additionalProperties": false, - "required": ["id", "title", "url", "source", "language", "published_at", - "excerpt", "chars", "truncated", "evidence_sha256", "on_topic"], + "required": ["id", "title", "url", "resolved_url", "publisher_host", "source", "language", + "published_at", "excerpt", "chars", "truncated", "evidence_sha256", "on_topic"], "properties": { "id": string(1, 64), "title": string(0, 400), "url": string(1, 2048), + "resolved_url": {"description": "Where the read ended after redirects (a Google News link's publisher page); null when unknown or still on the aggregator"}, + "publisher_host": {"description": "The publisher's host (resolved_url's, else url's unless an aggregator's), or null"}, "source": string(0, 200), "language": string(0, 16), "published_at": string(0, 40), diff --git a/crates/toolbox/src/peer.rs b/crates/toolbox/src/peer.rs index 76120646..b9968957 100644 --- a/crates/toolbox/src/peer.rs +++ b/crates/toolbox/src/peer.rs @@ -174,11 +174,12 @@ pub fn catalog(library: &Library) -> Vec { decls.push(decl( SEARCH, "read", - "Search free news sources (Google News, GDELT, publisher feeds) within this app's granted languages, regions, domains and recency; returns dated items with their sources, saved in this app's toolbox folder.".into(), + "Search free sources through the octos metasearch within this app's granted languages, regions, domains, categories and recency: category `news` (default: Google News, GDELT, publisher feeds, Hacker News), `general` (general web search: DuckDuckGo, Bing, Brave, Google, Wikipedia), `it` (GitHub, Hacker News, Stack Exchange), `science` (arXiv, OpenAlex), `social` (Mastodon; posts are not returned as evidence). Returns dated items with their sources, saved in this app's toolbox folder.".into(), json!({ "type": "object", "required": ["query"], "additionalProperties": false, "properties": { "query": {"type": "string", "minLength": 1, "maxLength": 300}, + "category": {"type": "string", "enum": ["news", "general", "it", "science", "social"], "description": "metasearch category (default news; must be granted to the app)"}, "lang": {"type": "string", "description": "BCP-47 language to search in (must be granted to the app)"}, "region": {"type": "string", "description": "ISO 3166-1 alpha-2 region"}, "max_age_days": {"type": "integer", "minimum": 1, "description": "only material this recent"}, @@ -414,8 +415,21 @@ impl PeerToolbox { let count = args["count"] .as_u64() .map_or(DEFAULT_SEARCH_COUNT, |c| c.clamp(1, 30) as u32); - let narrowed = scope::narrow_search(&app.scope, query, lang, region, max_age_hours, count) - .map_err(|e| PeerError::new("denied", e))?; + let category = args["category"] + .as_str() + .map(str::trim) + .filter(|c| !c.is_empty()) + .map(str::to_owned); + let narrowed = scope::narrow_search_in( + &app.scope, + category.as_deref().unwrap_or(scope::SEARCH_CATEGORY), + query, + lang, + region, + max_age_hours, + count, + ) + .map_err(|e| PeerError::new("denied", e))?; let ctx = self.context(app, SEARCH, 0); let found = self .backend @@ -428,6 +442,7 @@ impl PeerToolbox { limit: narrowed.limit, max_age_hours: narrowed.max_age_hours, max_fetches: MAX_SEARCH_FETCHES, + category: category.clone(), }, ) .await @@ -448,7 +463,7 @@ impl PeerToolbox { .collect(); let now = self.stamp(); let doc = json!({ - "kind": "search", "app_id": app.app_id, "query": query, "retrieved_at": now, + "kind": "search", "app_id": app.app_id, "query": query, "category": category.as_deref().unwrap_or(scope::SEARCH_CATEGORY), "retrieved_at": now, "items": items, "providers": found.providers, "partial": found.partial, "notes": notes, }); let file = save(&app.folder, "search", query, &doc)?; diff --git a/crates/toolbox/src/research/live.rs b/crates/toolbox/src/research/live.rs index cfe9a02e..f3ce5c38 100644 --- a/crates/toolbox/src/research/live.rs +++ b/crates/toolbox/src/research/live.rs @@ -1212,6 +1212,7 @@ impl ResearchBackend for InterimResearch { Ok(PageText { text, title: Some(article.title.to_string()).filter(|t| !t.is_empty()), + final_url: Some(final_url), }) }) } @@ -1397,6 +1398,7 @@ mod tests { limit: 5, max_age_hours: Some(72), max_fetches: 8, + category: None, } } diff --git a/crates/toolbox/src/research/mod.rs b/crates/toolbox/src/research/mod.rs index 982214db..b08edb3f 100644 --- a/crates/toolbox/src/research/mod.rs +++ b/crates/toolbox/src/research/mod.rs @@ -72,6 +72,10 @@ pub struct SearchQuery { /// Feeds and API responses the backend may fetch for this search, at /// most [`MAX_SEARCH_FETCHES`]. pub max_fetches: u32, + /// The metasearch category, when not the backend's default (`news`): + /// `general`, `it`, `social`, `science`. Already within the grant. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub category: Option, } /// One item a backend found. `via` names the provider. @@ -136,6 +140,11 @@ pub struct PageText { pub text: String, #[serde(default)] pub title: Option, + /// Where the read ended, after HTTP and browser (JavaScript) redirects: + /// for a Google News link, the publisher's page. `None` when the backend + /// cannot tell. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub final_url: Option, } /// Finds and reads sources. The fixture backend, the interim live adapter, @@ -315,11 +324,17 @@ impl ResearchHost { let region = input["region"].as_str().map(str::to_owned); let limit = input["limit"].as_u64().unwrap_or(5).clamp(1, 10) as u32; let requested_age = input["max_age_hours"].as_u64().map(|h| h as u32); + let category = input["category"] + .as_str() + .map(str::trim) + .filter(|c| !c.is_empty()) + .map(str::to_owned); // octos's `Scope::search_args` (or the thin parser's same rules): // a language, region or category outside the grant is refused, // recency and the result count are clamped. - let narrowed = scope::narrow_search( + let narrowed = scope::narrow_search_in( scope, + category.as_deref().unwrap_or(scope::SEARCH_CATEGORY), &topic, language.as_deref(), region.as_deref(), @@ -335,6 +350,7 @@ impl ResearchHost { limit, max_age_hours: narrowed.max_age_hours, max_fetches: MAX_SEARCH_FETCHES, + category, }; let topic = Topic::new(&query.topic); let mut results = self.backend.search(ctx, query.clone()).await?; @@ -380,6 +396,8 @@ impl ResearchHost { published_at: item.published_at.clone(), retrieved_at: queried_at.clone(), evidence_sha256: None, + resolved_url: None, + publisher_host: None, via: format!("search:{}", item.via), }); items.push(json!({ @@ -504,10 +522,13 @@ impl ResearchHost { } else { item.title.clone() }; + let (resolved_url, publisher_host) = resolve(&item.url, page.final_url.as_deref()); let output = json!({ "id": id, "title": clip(&title, 400), "url": item.url, + "resolved_url": resolved_url, + "publisher_host": publisher_host, "source": clip(&item.source, 200), "language": clip(&item.language, 16), "published_at": clip(&item.published_at, 40), @@ -526,6 +547,8 @@ impl ResearchHost { published_at: item.published_at.clone(), retrieved_at, evidence_sha256: Some(hash), + resolved_url, + publisher_host, via: "article".into(), }]; self.state(&ctx.run_id, |s| { @@ -1000,6 +1023,26 @@ fn host_of(url: &str) -> Option { (!host.is_empty()).then_some(host) } +/// Where an item's read ended (`resolved_url`) and its publisher's host. +/// A read that ended on an aggregator (still a Google News link) is not +/// resolved; the publisher host then comes from the item's own link unless +/// that is an aggregator's too. +fn resolve(url: &str, final_url: Option<&str>) -> (Option, Option) { + let not_aggregator = + |u: &&str| host_of(u).is_some_and(|h| !AGGREGATOR_HOSTS.contains(&h.as_str())); + let resolved = final_url + .map(str::trim) + .filter(|u| u.starts_with("http://") || u.starts_with("https://")) + .filter(not_aggregator) + .filter(|u| u.len() <= 2048) + .map(str::to_owned); + let host = resolved + .as_deref() + .or(Some(url).filter(not_aggregator)) + .and_then(host_of); + (resolved, host) +} + /// What identifies an item's publisher before it is read: its source name, /// and its link's host unless that is an aggregator's. fn publisher_keys(item: &FoundItem) -> Vec { @@ -1018,6 +1061,24 @@ fn publisher_keys(item: &FoundItem) -> Vec { mod tests { use super::*; + #[test] + fn google_news_links_resolve_only_off_the_aggregator() { + let gn = "https://news.google.com/rss/articles/CBMi?oc=5"; + assert_eq!(resolve(gn, None), (None, None)); + assert_eq!(resolve(gn, Some(gn)), (None, None)); + assert_eq!( + resolve(gn, Some("https://www.startupfortune.com/dhh/")), + ( + Some("https://www.startupfortune.com/dhh/".into()), + Some("startupfortune.com".into()) + ) + ); + assert_eq!( + resolve("https://bbc.co.uk/a", None), + (None, Some("bbc.co.uk".into())) + ); + } + #[test] fn refusals_mark_the_publisher_and_the_host_the_read_ended_on() { let bot = "bot_challenge: a DataDome challenge (not bypassed) (final URL: https://www.nytimes.com/a/b.html?x=1)"; diff --git a/crates/toolbox/src/research/octos.rs b/crates/toolbox/src/research/octos.rs index dc38b6dc..1d06eda7 100644 --- a/crates/toolbox/src/research/octos.rs +++ b/crates/toolbox/src/research/octos.rs @@ -433,7 +433,13 @@ impl ResearchBackend for OctosResearch { let now = chrono::Utc::now(); let scope = &ctx.app.scope; let scoped = scope - .search_args(&search_args(&query, &self.config.category), now) + .search_args( + &search_args( + &query, + query.category.as_deref().unwrap_or(&self.config.category), + ), + now, + ) .map_err(HostError::Denied)?; let request = search_request(&scoped, &self.config, now); let response = self.metasearch.search(&request).await; @@ -527,6 +533,7 @@ impl ResearchBackend for OctosResearch { Ok(PageText { text: page.text, title: page.meta.title.filter(|t| !t.trim().is_empty()), + final_url: Some(page.final_url), }) }) } @@ -545,11 +552,12 @@ impl ResearchBackend for OctosResearch { .map_err(HostError::Denied)?; let links = extract_links(&page.html, &page.final_url); Ok(LinkedPage { - final_url: page.final_url, page: PageText { text: page.text, title: page.meta.title.filter(|t| !t.trim().is_empty()), + final_url: Some(page.final_url.clone()), }, + final_url: page.final_url, links, }) }) @@ -702,6 +710,7 @@ mod tests { limit: 5, max_age_hours: Some(72), max_fetches: 8, + category: None, }; let now = chrono::Utc::now(); let scoped = scope @@ -842,6 +851,7 @@ mod tests { limit: 5, max_age_hours: None, max_fetches: 8, + category: None, }; let now = chrono::Utc::now(); let scoped = grant diff --git a/crates/toolbox/src/runner.rs b/crates/toolbox/src/runner.rs index e6b812f9..714d6377 100644 --- a/crates/toolbox/src/runner.rs +++ b/crates/toolbox/src/runner.rs @@ -533,7 +533,13 @@ pub async fn run( provenance_order .iter() .filter_map(|id| provenance.get(id)) - .filter(|p| referenced.contains(p.id.as_str()) || referenced.contains(p.url.as_str())) + .filter(|p| { + referenced.contains(p.id.as_str()) + || referenced.contains(p.url.as_str()) + || p.resolved_url + .as_deref() + .is_some_and(|u| referenced.contains(u)) + }) .cloned() .collect() } else { @@ -642,7 +648,10 @@ fn finish_output( schema .validate(&data) .map_err(|e| format!("output schema: {e}"))?; - let urls: BTreeSet<&str> = provenance.values().map(|p| p.url.as_str()).collect(); + let urls: BTreeSet<&str> = provenance + .values() + .flat_map(|p| std::iter::once(p.url.as_str()).chain(p.resolved_url.as_deref())) + .collect(); if let Some(bad) = json::strings(&data) .into_iter() .find(|s| json::contains_url(s) && !urls.contains(s)) diff --git a/crates/toolbox/src/scope.rs b/crates/toolbox/src/scope.rs index fb2c0387..28af2b4d 100644 --- a/crates/toolbox/src/scope.rs +++ b/crates/toolbox/src/scope.rs @@ -116,7 +116,7 @@ pub struct NarrowedSearch { /// Narrows one search to the grant, or refuses it with the reason: a /// language, region or category outside the grant is refused; recency and /// the result count are clamped. With the feature this is -/// `Scope::search_args`. +/// `Scope::search_args`. Searches [`SEARCH_CATEGORY`]. pub fn narrow_search( scope: &Scope, topic: &str, @@ -124,11 +124,34 @@ pub fn narrow_search( region: Option<&str>, max_age_hours: Option, limit: u32, +) -> Result { + narrow_search_in( + scope, + SEARCH_CATEGORY, + topic, + language, + region, + max_age_hours, + limit, + ) +} + +/// [`narrow_search`] in another metasearch category (`general`, `it`, +/// `social`, `science`, `news`), which must be in the grant's `categories` +/// (an empty list allows every category). +pub fn narrow_search_in( + scope: &Scope, + category: &str, + topic: &str, + language: Option<&str>, + region: Option<&str>, + max_age_hours: Option, + limit: u32, ) -> Result { #[cfg(feature = "octos-engine")] { let now = chrono::Utc::now(); - let mut args = json!({"query": topic, "category": SEARCH_CATEGORY, "count": limit}); + let mut args = json!({"query": topic, "category": category, "count": limit}); if let Some(language) = language.filter(|l| !l.is_empty()) { args["lang"] = json!([language]); } @@ -149,7 +172,15 @@ pub fn narrow_search( }) } #[cfg(not(feature = "octos-engine"))] - compat::narrow_search(scope, topic, language, region, max_age_hours, limit) + compat::narrow_search_in( + scope, + category, + topic, + language, + region, + max_age_hours, + limit, + ) } /// A `deep_crawl` narrowed to the grant (the `crawl` capability's limits). diff --git a/crates/toolbox/src/scope/compat.rs b/crates/toolbox/src/scope/compat.rs index e127083d..93c82498 100644 --- a/crates/toolbox/src/scope/compat.rs +++ b/crates/toolbox/src/scope/compat.rs @@ -177,6 +177,27 @@ pub fn narrow_search( region: Option<&str>, max_age_hours: Option, limit: u32, +) -> Result { + narrow_search_in( + scope, + SEARCH_CATEGORY, + topic, + language, + region, + max_age_hours, + limit, + ) +} + +/// [`narrow_search`] in another metasearch category. +pub fn narrow_search_in( + scope: &Scope, + category: &str, + topic: &str, + language: Option<&str>, + region: Option<&str>, + max_age_hours: Option, + limit: u32, ) -> Result { if topic.trim().is_empty() { return Err("query is required".into()); @@ -206,9 +227,12 @@ pub fn narrow_search( max_age_hours = Some(floor); } } - if !scope.categories.is_empty() && !scope.categories.iter().any(|c| c == SEARCH_CATEGORY) { + if !CATEGORIES.contains(&category) { + return Err(format!("unknown category {category:?}")); + } + if !scope.categories.is_empty() && !scope.categories.iter().any(|c| c == category) { return Err(format!( - "category {SEARCH_CATEGORY} is not in this app's research grant" + "category {category} is not in this app's research grant" )); } let asked = limit.max(1) as usize; diff --git a/crates/toolbox/templates/dossier/fixtures/failed-read-and-empty-question.json b/crates/toolbox/templates/dossier/fixtures/failed-read-and-empty-question.json new file mode 100644 index 00000000..16a544bd --- /dev/null +++ b/crates/toolbox/templates/dossier/fixtures/failed-read-and-empty-question.json @@ -0,0 +1,182 @@ +{ + "name": "failed-read-and-empty-question", + "params": { + "categories": [ + "general" + ], + "language": "zh", + "questions": [ + { + "queries": [ + { + "language": "en", + "terms": "Lumen desktop maintainers" + } + ], + "question": "Who maintains the Lumen desktop?" + }, + { + "queries": [ + { + "language": "en", + "terms": "Lumen desktop BSD" + } + ], + "question": "Does Lumen desktop run on BSD?" + } + ], + "read_top": 1 + }, + "fixture": { + "now": "2026-09-20T08:00:00Z", + "searches": [ + { + "topic": "Lumen desktop maintainers", + "language": "en", + "delay_ms": 30, + "partial": false, + "providers": [ + "wikipedia", + "google_news" + ], + "items": [ + { + "url": "https://example.invalid/blocked/lumen-team", + "title": "Lumen desktop maintainers named", + "source": "Walled Times", + "language": "en", + "published_at": "2025-01-05T00:00:00Z", + "via": "" + }, + { + "url": "https://example.invalid/news/lumen-team", + "title": "Lumen desktop maintainers list grows", + "source": "Harbor Daily", + "language": "en", + "published_at": "2025-02-01T00:00:00Z", + "via": "" + } + ], + "error": null + }, + { + "topic": "Lumen desktop BSD", + "language": "en", + "delay_ms": 30, + "partial": false, + "providers": [ + "wikipedia", + "google_news" + ], + "items": [], + "error": "no search engine answered: wikipedia (en) timed out" + } + ], + "pages": { + "https://example.invalid/blocked/lumen-team": { + "text": "", + "title": null, + "delay_ms": 40, + "error": "paywall: subscribe to read" + }, + "https://example.invalid/news/lumen-team": { + "text": "The Lumen desktop maintainers now number five, led by Ada Park. The maintainers review every change to the Lumen desktop.", + "title": null, + "delay_ms": 40, + "error": null + } + }, + "translations": {}, + "model": { + "delay_ms": 0, + "fail": false, + "inject_url": false, + "cite_unknown": false + } + }, + "expected": { + "status": "partial", + "reasons": [ + "question 2: 1 of 1 searches failed", + "question 2: nothing readable was found" + ], + "data": { + "language": "zh", + "missing": 1, + "questions": [ + { + "digest": { + "dropped_points": 0, + "language": "zh", + "off_topic": [], + "points": [ + { + "citations": [ + "s65803035137e" + ], + "text": "The Lumen desktop maintainers now number five, led by Ada Park." + } + ], + "summary": "Sources: Lumen desktop maintainers list grows", + "summary_check": { + "dropped": 0, + "flagged": 0, + "sentences": 1 + }, + "task": "digest" + }, + "failed": 1, + "found": 2, + "left": 0, + "off_topic": 0, + "question": "Who maintains the Lumen desktop?", + "read": 1, + "searches": [ + { + "answered": true, + "category": "general", + "found": 2, + "language": "en", + "terms": "Lumen desktop maintainers" + } + ], + "sources": [ + { + "evidence_sha256": "0ac6622e1217cb71c23ed901c2462d584e3d54673ef16f81d546f794110167a1", + "id": "s65803035137e", + "language": "en", + "published_at": "2025-02-01T00:00:00Z", + "publisher_host": "example.invalid", + "resolved_url": null, + "source": "Harbor Daily", + "title": "Lumen desktop maintainers list grows", + "url": "https://example.invalid/news/lumen-team" + } + ], + "unreadable": 0 + }, + { + "digest": null, + "failed": 0, + "found": 0, + "left": 0, + "off_topic": 0, + "question": "Does Lumen desktop run on BSD?", + "read": 0, + "searches": [ + { + "answered": false, + "category": "general", + "found": 0, + "language": "en", + "terms": "Lumen desktop BSD" + } + ], + "sources": [], + "unreadable": 0 + } + ], + "searches_skipped": 0 + } + } +} diff --git a/crates/toolbox/templates/dossier/fixtures/two-questions.json b/crates/toolbox/templates/dossier/fixtures/two-questions.json new file mode 100644 index 00000000..b49d3e2e --- /dev/null +++ b/crates/toolbox/templates/dossier/fixtures/two-questions.json @@ -0,0 +1,329 @@ +{ + "name": "two-questions", + "params": { + "categories": [ + "general", + "news" + ], + "language": "en", + "questions": [ + { + "queries": [ + { + "language": "en", + "terms": "Lumen desktop" + }, + { + "language": "zh", + "terms": "Lumen 桌面" + } + ], + "question": "What is the Lumen desktop and who started it?" + }, + { + "queries": [ + { + "language": "en", + "terms": "Lumen desktop release" + } + ], + "question": "When was Lumen desktop first released?" + } + ], + "read_budget": 6, + "read_top": 2 + }, + "fixture": { + "now": "2026-09-20T08:00:00Z", + "searches": [ + { + "topic": "Lumen desktop", + "language": "en", + "delay_ms": 30, + "partial": false, + "providers": [ + "wikipedia", + "google_news" + ], + "items": [ + { + "url": "https://example.invalid/wiki/lumen", + "title": "Lumen (desktop environment)", + "source": "Example Wiki", + "language": "en", + "published_at": "2024-03-01T00:00:00Z", + "via": "" + }, + { + "url": "https://example.invalid/news/lumen-launch", + "title": "Developer Ada Park launches Lumen desktop", + "source": "Harbor Daily", + "language": "en", + "published_at": "2024-06-10T09:00:00Z", + "via": "" + } + ], + "error": null + }, + { + "topic": "Lumen 桌面", + "language": "zh", + "delay_ms": 30, + "partial": false, + "providers": [ + "wikipedia", + "google_news" + ], + "items": [ + { + "url": "https://example.invalid/zh/lumen", + "title": "Lumen 桌面环境简介", + "source": "示例科技", + "language": "zh", + "published_at": "2024-07-02T00:00:00Z", + "via": "" + } + ], + "error": null + }, + { + "topic": "Lumen desktop release", + "language": "en", + "delay_ms": 30, + "partial": false, + "providers": [ + "wikipedia", + "google_news" + ], + "items": [ + { + "url": "https://news.google.com/rss/articles/CBMiLumen10?oc=5", + "title": "Lumen desktop 1.0 release arrives", + "source": "Tech Courier", + "language": "en", + "published_at": "2024-06-10T12:00:00Z", + "via": "" + }, + { + "url": "https://example.invalid/wiki/lumen", + "title": "Lumen (desktop environment)", + "source": "Example Wiki", + "language": "en", + "published_at": "2024-03-01T00:00:00Z", + "via": "" + } + ], + "error": null + } + ], + "pages": { + "https://example.invalid/news/lumen-launch": { + "text": "Developer Ada Park launched the Lumen desktop on 10 June 2024. Park said the Lumen desktop grew out of personal configuration files shared since 2023.", + "title": null, + "delay_ms": 40, + "error": null + }, + "https://example.invalid/wiki/lumen": { + "text": "Lumen is a desktop environment for Linux started by developer Ada Park in 2024. The Lumen desktop focuses on keyboard-driven tiling windows. Its first public release was version 1.0 in June 2024.", + "title": null, + "delay_ms": 40, + "error": null + }, + "https://example.invalid/zh/lumen": { + "text": "Lumen 桌面是一个面向 Linux 的桌面环境,由开发者 Ada Park 于 2024 年发起。Lumen 桌面以键盘操作为主。", + "title": null, + "delay_ms": 40, + "error": null + }, + "https://news.google.com/rss/articles/CBMiLumen10?oc=5": { + "text": "The Lumen desktop 1.0 release arrived on 10 June 2024 with an installer for Arch Linux. The release of Lumen desktop drew 2,000 stars in its first week.", + "title": null, + "delay_ms": 40, + "final_url": "https://www.techcourier.example/2024/06/lumen-1-0", + "error": null + } + }, + "translations": {}, + "model": { + "delay_ms": 0, + "fail": false, + "inject_url": false, + "cite_unknown": false + } + }, + "expected": { + "status": "ready", + "reasons": [], + "data": { + "language": "en", + "missing": 0, + "questions": [ + { + "digest": { + "dropped_points": 0, + "language": "en", + "off_topic": [], + "points": [ + { + "citations": [ + "s3564eaa47d5a" + ], + "text": "Lumen is a desktop environment for Linux started by developer Ada Park in 2024." + }, + { + "citations": [ + "se80a28f6cf9d" + ], + "text": "Lumen 桌面是一个面向 Linux 的桌面环境,由开发者 Ada Park 于 2024 年发起。" + } + ], + "summary": "Sources: Lumen (desktop environment); Lumen 桌面环境简介", + "summary_check": { + "dropped": 0, + "flagged": 0, + "sentences": 1 + }, + "task": "digest" + }, + "failed": 0, + "found": 3, + "left": 1, + "off_topic": 0, + "question": "What is the Lumen desktop and who started it?", + "read": 2, + "searches": [ + { + "answered": true, + "category": "general", + "found": 2, + "language": "en", + "terms": "Lumen desktop" + }, + { + "answered": true, + "category": "news", + "found": 2, + "language": "en", + "terms": "Lumen desktop" + }, + { + "answered": true, + "category": "general", + "found": 1, + "language": "zh", + "terms": "Lumen 桌面" + }, + { + "answered": true, + "category": "news", + "found": 1, + "language": "zh", + "terms": "Lumen 桌面" + } + ], + "sources": [ + { + "evidence_sha256": "f58c6d6e0311ced6d366521fdc4304eea9eacb08c6598c92db9de087694f5bd3", + "id": "s3564eaa47d5a", + "language": "en", + "published_at": "2024-03-01T00:00:00Z", + "publisher_host": "example.invalid", + "resolved_url": null, + "source": "Example Wiki", + "title": "Lumen (desktop environment)", + "url": "https://example.invalid/wiki/lumen" + }, + { + "evidence_sha256": "7c423a4489653cc9124ace5b07884fd80f4f334134a66eb902ca3e715c47cbfd", + "id": "se80a28f6cf9d", + "language": "zh", + "published_at": "2024-07-02T00:00:00Z", + "publisher_host": "example.invalid", + "resolved_url": null, + "source": "示例科技", + "title": "Lumen 桌面环境简介", + "url": "https://example.invalid/zh/lumen" + } + ], + "unreadable": 0 + }, + { + "digest": { + "dropped_points": 0, + "language": "en", + "off_topic": [], + "points": [ + { + "citations": [ + "s590de35722df" + ], + "text": "The Lumen desktop 1.0 release arrived on 10 June 2024 with an installer for Arch Linux." + }, + { + "citations": [ + "s3564eaa47d5a" + ], + "text": "Lumen is a desktop environment for Linux started by developer Ada Park in 2024." + } + ], + "summary": "Sources: Lumen desktop 1.0 release arrives; Lumen (desktop environment)", + "summary_check": { + "dropped": 0, + "flagged": 0, + "sentences": 1 + }, + "task": "digest" + }, + "failed": 0, + "found": 2, + "left": 0, + "off_topic": 0, + "question": "When was Lumen desktop first released?", + "read": 2, + "searches": [ + { + "answered": true, + "category": "general", + "found": 2, + "language": "en", + "terms": "Lumen desktop release" + }, + { + "answered": true, + "category": "news", + "found": 2, + "language": "en", + "terms": "Lumen desktop release" + } + ], + "sources": [ + { + "evidence_sha256": "4393f82c52ba4bdf2fc7483635526580b66423509b32214f3261add13aaa2fd5", + "id": "s590de35722df", + "language": "en", + "published_at": "2024-06-10T12:00:00Z", + "publisher_host": "techcourier.example", + "resolved_url": "https://www.techcourier.example/2024/06/lumen-1-0", + "source": "Tech Courier", + "title": "Lumen desktop 1.0 release arrives", + "url": "https://news.google.com/rss/articles/CBMiLumen10?oc=5" + }, + { + "evidence_sha256": "f58c6d6e0311ced6d366521fdc4304eea9eacb08c6598c92db9de087694f5bd3", + "id": "s3564eaa47d5a", + "language": "en", + "published_at": "2024-03-01T00:00:00Z", + "publisher_host": "example.invalid", + "resolved_url": null, + "source": "Example Wiki", + "title": "Lumen (desktop environment)", + "url": "https://example.invalid/wiki/lumen" + } + ], + "unreadable": 0 + } + ], + "searches_skipped": 0 + } + } +} diff --git a/crates/toolbox/templates/dossier/template.json b/crates/toolbox/templates/dossier/template.json new file mode 100644 index 00000000..9325ea83 --- /dev/null +++ b/crates/toolbox/templates/dossier/template.json @@ -0,0 +1,72 @@ +{ + "id": "dossier", + "version": "1.1.0", + "title": "Background dossier on a few questions", + "description": "Background research, not news: for each question, run its sub-queries (the caller writes them, in each language it wants, typically English and Chinese) in each requested metasearch category (general, it, news, social, science) over a long window (max_age_days, up to 1500), all searches started together; pool each question's readable results in rank order round-robin across its searches, skipping results the backend cannot read and preferring publishers with no failed read in this run; read up to read_top articles per question in rounds, replacing a failed or off-topic read with the next candidate, within read_budget attempts for the whole run (an article another question already read is reused, not read again); then one digest per question, in the reader's language, citing only articles read in this run, with the articles the model finds off the question left out. Ready when every question has a digest that kept every point; partial otherwise, with the reasons; failed when no question has anything read.", + "params": { + "type": "object", + "additionalProperties": false, + "required": ["questions", "language"], + "properties": { + "questions": {"type": "array", "minItems": 1, "maxItems": 6, "description": "The questions, each with its own search sub-queries.", "items": { + "type": "object", "additionalProperties": false, "required": ["question", "queries"], + "properties": { + "question": {"type": "string", "minLength": 1, "maxLength": 280, "description": "The question, in the reader's words; the digest's focus."}, + "queries": {"type": "array", "minItems": 1, "maxItems": 6, "description": "Search terms for this question, each in one language (write the Chinese terms in Chinese).", "items": { + "type": "object", "additionalProperties": false, "required": ["terms", "language"], + "properties": { + "terms": {"type": "string", "minLength": 1, "maxLength": 160}, + "language": {"type": "string", "minLength": 2, "maxLength": 16} + }}} + }}}, + "language": {"type": "string", "minLength": 2, "maxLength": 16, "description": "BCP 47 tag of the digests' language."}, + "categories": {"type": "array", "minItems": 1, "maxItems": 5, "default": ["general", "it", "news", "social"], "description": "Metasearch categories each sub-query is searched in (each must be in the app's grant). `social` finds posts, which are never evidence; it helps only when a post links an article.", "items": {"type": "string", "enum": ["general", "it", "news", "social", "science"]}}, + "per_search": {"type": "integer", "minimum": 1, "maximum": 10, "default": 6, "description": "Results asked of each search."}, + "read_top": {"type": "integer", "minimum": 1, "maximum": 8, "default": 4, "description": "Articles to read (and digest) per question."}, + "read_budget": {"type": "integer", "minimum": 1, "maximum": 24, "default": 12, "description": "Read attempts for the whole run, failed ones included (the app's own budget may be lower)."}, + "max_age_days": {"type": "integer", "minimum": 1, "maximum": 1500, "default": 1500, "description": "Oldest material, in days (clamped to the app's grant)."} + } + }, + "modules": [{"module": "research", "methods": ["search", "article", "digest"]}], + "budget": {"max_calls": 64, "max_model_calls": 6, "max_reads": 24, "max_ms": 240000, "max_concurrency": 6}, + "output": { + "type": "object", + "additionalProperties": false, + "required": ["language", "questions", "searches_skipped", "missing"], + "properties": { + "language": {"type": "string"}, + "questions": {"type": "array", "maxItems": 6, "items": { + "type": "object", "additionalProperties": false, + "required": ["question", "searches", "found", "unreadable", "read", "off_topic", "failed", "left", "digest", "sources"], + "properties": { + "question": {"type": "string"}, + "searches": {"type": "array", "maxItems": 30, "items": { + "type": "object", "additionalProperties": false, "required": ["terms", "language", "category", "found", "answered"], + "properties": { + "terms": {"type": "string"}, "language": {"type": "string"}, "category": {"type": "string"}, + "found": {"type": "integer", "minimum": 0}, "answered": {"type": "boolean", "description": "false when the search failed"} + }}}, + "found": {"type": "integer", "minimum": 0, "description": "Distinct results across this question's searches."}, + "unreadable": {"type": "integer", "minimum": 0}, + "read": {"type": "integer", "minimum": 0, "description": "Articles kept for this question."}, + "off_topic": {"type": "integer", "minimum": 0}, + "failed": {"type": "integer", "minimum": 0}, + "left": {"type": "integer", "minimum": 0, "description": "Readable candidates not tried."}, + "digest": {"description": "research.digest's result for this question, or null when nothing was read or the model failed."}, + "sources": {"type": "array", "maxItems": 8, "items": { + "type": "object", "additionalProperties": false, + "required": ["id", "title", "url", "source", "language", "published_at", "evidence_sha256"], + "properties": { + "id": {"type": "string"}, "title": {"type": "string"}, "url": {"type": "string"}, + "source": {"type": "string"}, "language": {"type": "string"}, "published_at": {"type": "string"}, + "evidence_sha256": {"type": "string", "minLength": 64, "maxLength": 64}, + "resolved_url": {"description": "Where the read ended after redirects (a Google News link's publisher page); null while unresolved. Cite this, not url, when present."}, + "publisher_host": {"description": "The publisher's host, or null when unknown."} + }}} + }}}, + "searches_skipped": {"type": "integer", "minimum": 0, "description": "Planned searches not run because of the run's search cap (30)."}, + "missing": {"type": "integer", "minimum": 0, "description": "Reads that failed."} + } + }, + "provenance": true +} diff --git a/crates/toolbox/templates/dossier/template.octoscript b/crates/toolbox/templates/dossier/template.octoscript new file mode 100644 index 00000000..9ef141a4 --- /dev/null +++ b/crates/toolbox/templates/dossier/template.octoscript @@ -0,0 +1,311 @@ +use mod.research +use mod.std.array + +// Dossier: background research on a few questions. Each question brings its +// own sub-queries (in the languages the caller chose); each sub-query is +// searched in each requested category over a long window, all searches +// started together. Each question's readable results are pooled in rank +// order, round-robin across its searches, read in rounds up to `read_top`, +// and digested once with citations. + +// 1. Plan the searches: each question gets an equal share of the run's +// search cap (30), filled query by query, category by category. Every +// search starts before any is awaited. +let max_searches = 30 +let nq = array.len(request.questions) +let cap_each = 0 +while (cap_each + 1) * nq <= max_searches { + cap_each = cap_each + 1 +} +let searches = [] +let meta = [] +let owners = [] +let skipped = 0 +let reasons = [] +let qi = 0 +for q in request.questions { + let planned = 0 + for query in q.queries { + for category in request.categories { + if planned < cap_each { + array.push(meta, {terms: query.terms, language: query.language, category: category, found: 0, answered: true}) + array.push(owners, qi) + array.push(searches, research.search({topic: query.terms, language: query.language, category: category, limit: request.per_search, max_age_hours: request.max_age_days * 24})) + planned = planned + 1 + } else { + skipped = skipped + 1 + } + } + } + qi = qi + 1 +} +if skipped > 0 { + array.push(reasons, skipped + " planned search(es) not run: at most " + cap_each + " searches per question") +} + +// 2. Await the searches. Each search's readable results, in the host's +// order, with their publishers; results the backend cannot read are +// counted and skipped. +let lists = [] +let list_sources = [] +let found_ids = [] +let unreadable = [] +for i in array.range(0, nq) { + array.push(found_ids, []) + array.push(unreadable, 0) +} +let si = 0 +for pending in searches { + let found = try pending.await() catch nil + let ids = [] + let sources = [] + let owner = owners[si] + if found == nil { + meta[si].answered = false + } else { + meta[si].found = array.len(found.items) + for item in found.items { + if array.contains(found_ids[owner], item.id) == false { + array.push(found_ids[owner], item.id) + if item.readable { + array.push(ids, item.id) + array.push(sources, item.source) + } else { + unreadable[owner] = unreadable[owner] + 1 + } + } + } + } + array.push(lists, ids) + array.push(list_sources, sources) + si = si + 1 +} + +// 3. Each question's pool: rank 1 of each of its searches, then rank 2, +// and so on, each article once. +let pools = [] +let pool_sources = [] +for q in array.range(0, nq) { + let pool = [] + let pool_source = [] + let depth = 0 + let more = true + while more { + more = false + si = 0 + for ids in lists { + if owners[si] == q && depth < array.len(ids) { + more = true + if array.contains(pool, ids[depth]) == false { + array.push(pool, ids[depth]) + array.push(pool_source, list_sources[si][depth]) + } + } + si = si + 1 + } + depth = depth + 1 + } + array.push(pools, pool) + array.push(pool_sources, pool_source) +} + +// 4. Read in rounds (at most four). Each round starts, for every question +// still short of `read_top`, reads of its next untried candidates, the +// first one from a publisher with no failed read in this run when there is +// one. An article another question already read is reused without a read; +// one that failed or was off topic is skipped. All read attempts together +// stay within `read_budget`. +let kept = [] +let kept_sources = [] +let tried = [] +let failed = [] +let off_topic = [] +for q in array.range(0, nq) { + array.push(kept, []) + array.push(kept_sources, []) + array.push(tried, []) + array.push(failed, 0) + array.push(off_topic, 0) +} +let read_ok = [] +let read_records = [] +let read_bad = [] +let failed_publishers = [] +let attempts = 0 +let missing = 0 +let round = 0 +while round < 4 { + let batch = [] + let batch_q = [] + let batch_ids = [] + let batch_publishers = [] + for q in array.range(0, nq) { + let want = request.read_top - array.len(kept[q]) + let looking = true + while want > 0 && looking { + let pick = nil + let fallback = nil + let i = 0 + for id in pools[q] { + if array.contains(tried[q], id) == false { + if fallback == nil { + fallback = i + } + if pick == nil && array.contains(failed_publishers, pool_sources[q][i]) == false { + pick = i + } + } + i = i + 1 + } + if pick == nil { + pick = fallback + } + if pick == nil { + looking = false + } elif array.contains(batch_ids, pools[q][pick]) { + // Being read for another question this round: reuse it in + // the next round. + looking = false + } else { + let id = pools[q][pick] + array.push(tried[q], id) + let done = -1 + let j = 0 + for known in read_ok { + if known == id { + done = j + } + j = j + 1 + } + if done >= 0 { + array.push(kept[q], id) + array.push(kept_sources[q], read_records[done]) + want = want - 1 + } elif array.contains(read_bad, id) { + // Failed or off topic for another question. + } elif attempts < request.read_budget { + array.push(batch, research.article({id: id})) + array.push(batch_q, q) + array.push(batch_ids, id) + array.push(batch_publishers, pool_sources[q][pick]) + attempts = attempts + 1 + want = want - 1 + } else { + looking = false + } + } + } + } + if array.len(batch) == 0 { + break + } + let b = 0 + for pending in batch { + let q = batch_q[b] + let article = try pending.await() catch nil + if article == nil { + missing = missing + 1 + failed[q] = failed[q] + 1 + array.push(read_bad, batch_ids[b]) + let publisher = batch_publishers[b] + if publisher != "" && array.contains(failed_publishers, publisher) == false { + array.push(failed_publishers, publisher) + } + } elif article.on_topic == false { + off_topic[q] = off_topic[q] + 1 + array.push(read_bad, batch_ids[b]) + } else { + let record = {id: article.id, title: article.title, url: article.url, resolved_url: article.resolved_url, publisher_host: article.publisher_host, source: article.source, language: article.language, published_at: article.published_at, evidence_sha256: article.evidence_sha256} + array.push(read_ok, article.id) + array.push(read_records, record) + array.push(kept[q], article.id) + array.push(kept_sources[q], record) + } + b = b + 1 + } + round = round + 1 +} + +// 5. One digest per question, all started together, focused on the +// question: the model also names the articles not about it, which leave +// its sources. +let digests = [] +for q in array.range(0, nq) { + if array.len(kept[q]) > 0 { + array.push(digests, research.digest({task: "digest", language: request.language, article_ids: kept[q], focus: request.questions[q].question})) + } else { + array.push(digests, nil) + } +} +let out = [] +let any_read = false +for q in array.range(0, nq) { + let question = request.questions[q].question + let digest = nil + if digests[q] != nil { + let got = try digests[q].await() catch nil + digest = got + } + let sources = kept_sources[q] + if digest != nil { + let left_in = [] + for source in sources { + if array.contains(digest.off_topic, source.id) { + off_topic[q] = off_topic[q] + 1 + } else { + array.push(left_in, source) + } + } + sources = left_in + } + let q_searches = [] + let failed_searches = 0 + si = 0 + for m in meta { + if owners[si] == q { + array.push(q_searches, m) + if m.answered == false { + failed_searches = failed_searches + 1 + } + } + si = si + 1 + } + let left = 0 + for id in pools[q] { + if array.contains(tried[q], id) == false { + left = left + 1 + } + } + let label = "question " + (q + 1) + if failed_searches > 0 { + array.push(reasons, label + ": " + failed_searches + " of " + array.len(q_searches) + " searches failed") + } + if array.len(kept[q]) == 0 { + if array.len(pools[q]) == 0 { + array.push(reasons, label + ": nothing readable was found") + } else { + array.push(reasons, label + ": no article read (" + failed[q] + " failed, " + off_topic[q] + " off topic, " + left + " left untried)") + } + } elif digest == nil { + array.push(reasons, label + ": the digest failed; the sources are listed without it") + } elif digest.dropped_points > 0 { + array.push(reasons, label + ": the digest dropped " + digest.dropped_points + " invalid point(s)") + } + if array.len(kept[q]) > 0 && array.len(kept[q]) < request.read_top && left > 0 { + array.push(reasons, label + ": read " + array.len(kept[q]) + " of " + request.read_top + " while " + left + " readable candidates remained (read budget)") + } + if array.len(sources) > 0 { + any_read = true + } + array.push(out, {question: question, searches: q_searches, found: array.len(found_ids[q]), unreadable: unreadable[q], read: array.len(sources), off_topic: off_topic[q], failed: failed[q], left: left, digest: digest, sources: sources}) +} + +let status = "ready" +if array.len(reasons) > 0 { + status = "partial" +} +if any_read == false { + status = "failed" + array.push(reasons, "no question has an article read on it") +} +{status: status, reasons: reasons, data: {language: request.language, questions: out, searches_skipped: skipped, missing: missing}} diff --git a/crates/toolbox/templates/library.lock.json b/crates/toolbox/templates/library.lock.json index 0ffdb2a2..819e59e4 100644 --- a/crates/toolbox/templates/library.lock.json +++ b/crates/toolbox/templates/library.lock.json @@ -9,6 +9,10 @@ "version": "1.1.0", "digest": "a62e9457387aa33a6165638c61381442f2a78bdf57b8ab30b49f28c841770cc6" }, + "dossier": { + "version": "1.1.0", + "digest": "5bfbb70e5d5a91f6ca3eb53555c8e3879f0161d9d5394efeb081482eb24e1d34" + }, "market-brief": { "version": "1.1.0", "digest": "b9886ab78bf69459bf7c56720ee33871f026175b0870b746d5cc42314e953754" diff --git a/crates/toolbox/tests/forks.rs b/crates/toolbox/tests/forks.rs index 5616973d..37df4d11 100644 --- a/crates/toolbox/tests/forks.rs +++ b/crates/toolbox/tests/forks.rs @@ -297,6 +297,7 @@ async fn the_tool_surface_lists_runs_forks_and_evaluates() { [ "briefing", "compare", + "dossier", "market-brief", "news-digest", "topic-brief", diff --git a/crates/toolbox/tests/peer.rs b/crates/toolbox/tests/peer.rs index ac25e867..d255c001 100644 --- a/crates/toolbox/tests/peer.rs +++ b/crates/toolbox/tests/peer.rs @@ -53,6 +53,7 @@ impl ResearchBackend for Site { page: PageText { text: format!("The page at {url}."), title: Some(url.to_owned()), + final_url: None, }, links: links.iter().map(|l| l.to_string()).collect(), }) diff --git a/crates/toolbox/tests/runner.rs b/crates/toolbox/tests/runner.rs index fd790802..94849033 100644 --- a/crates/toolbox/tests/runner.rs +++ b/crates/toolbox/tests/runner.rs @@ -5,13 +5,18 @@ mod common; use common::{app, case, probe, temp_dir, template}; -use octosense_toolbox::fixture::{self, FixtureData}; +use octosense_toolbox::fixture::{self, FakeModel, FakeModelConfig, FixtureData}; use octosense_toolbox::host::{CallContext, HostError, HostFuture, HostReply, Usage}; +use octosense_toolbox::research::{ + FoundItem, PageText, ResearchBackend, ResearchHost, SearchQuery, SearchResults, +}; use octosense_toolbox::{ run, scope, AppContext, Budget, ErrorKind, RunOptions, RunStatus, ToolboxHost, }; use serde_json::{json, Value}; use std::cell::RefCell; +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; fn city() -> ( octosense_toolbox::fixture::FixtureCase, @@ -1313,3 +1318,147 @@ async fn a_template_states_its_status_and_reasons() { .await; assert_eq!(result.status, RunStatus::Failed); } + +/// A backend that records the queries it was asked and serves one Google +/// News item whose read ends on the publisher's page. +struct Probe { + queries: Mutex>, +} + +impl Probe { + fn host(self: &Arc) -> ResearchHost { + ResearchHost::new( + Arc::clone(self) as Arc, + Arc::new(FakeModel::new(BTreeMap::new(), FakeModelConfig::default())), + ) + } +} + +impl ResearchBackend for Probe { + fn search<'a>( + &'a self, + _ctx: &'a CallContext, + query: SearchQuery, + ) -> HostFuture<'a, Result> { + Box::pin(async move { + self.queries.lock().unwrap().push(query); + Ok(SearchResults { + items: vec![FoundItem { + url: "https://news.google.com/rss/articles/CBMiZmlzaW9u?oc=5".into(), + title: "Fusion energy milestone".into(), + source: "Example Post".into(), + language: "en".into(), + published_at: "2026-09-28T00:00:00Z".into(), + via: "fixture".into(), + readable: true, + snippet: "Fusion energy progress".into(), + }], + ..SearchResults::default() + }) + }) + } + fn read<'a>( + &'a self, + _ctx: &'a CallContext, + _item: &'a FoundItem, + ) -> HostFuture<'a, Result> { + Box::pin(async { + Ok(PageText { + text: "Fusion energy researchers report a milestone.".into(), + title: Some("Fusion energy milestone".into()), + final_url: Some("https://examplepost.com/fusion-energy".into()), + }) + }) + } +} + +#[tokio::test] +async fn a_search_in_an_ungranted_category_is_denied() { + let template = probe( + &["search"], + "use mod.research\nlet found = try research.search({topic: \"fusion energy\", category: \"science\"}).await() catch nil\n{status: \"partial\", data: {searched: found != nil}}\n", + ) + .unwrap(); + let folder = temp_dir("category-denied"); + let scoped = app(&folder).with_scope(scope::parse(&json!({"categories": ["news"]})).unwrap()); + let backend = Arc::new(Probe { + queries: Mutex::new(Vec::new()), + }); + let result = run( + &template, + &scoped, + json!({}), + &backend.host(), + RunOptions::default(), + ) + .await + .unwrap(); + assert_eq!(result.status, RunStatus::Partial); + assert_eq!(result.data["searched"], false); + assert!(result + .diagnostics + .iter() + .any(|d| d.contains("category science is not in this app's research grant"))); + assert!(backend.queries.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn a_granted_category_reaches_the_backend() { + let template = probe( + &["search"], + "use mod.research\nlet found = research.search({topic: \"fusion energy\", category: \"science\"}).await()\n{status: \"ready\", data: {n: found.items}}\n", + ) + .unwrap(); + let folder = temp_dir("category-granted"); + let scoped = + app(&folder).with_scope(scope::parse(&json!({"categories": ["science"]})).unwrap()); + let backend = Arc::new(Probe { + queries: Mutex::new(Vec::new()), + }); + let result = run( + &template, + &scoped, + json!({}), + &backend.host(), + RunOptions::default(), + ) + .await + .unwrap(); + assert_eq!(result.status, RunStatus::Ready); + let queries = backend.queries.lock().unwrap(); + assert_eq!(queries.len(), 1); + assert_eq!(queries[0].category.as_deref(), Some("science")); +} + +#[tokio::test] +async fn a_redirected_read_reports_its_resolved_url_and_publisher() { + let template = probe( + &["search", "article"], + "use mod.research\nlet found = research.search({topic: \"fusion energy\", category: \"science\"}).await()\nlet a = research.article({id: found.items[0].id}).await()\n{status: \"ready\", data: {resolved: a.resolved_url, publisher: a.publisher_host}}\n", + ) + .unwrap(); + let folder = temp_dir("resolved-url"); + let backend = Arc::new(Probe { + queries: Mutex::new(Vec::new()), + }); + let result = run( + &template, + &app(&folder), + json!({}), + &backend.host(), + RunOptions::default(), + ) + .await + .unwrap(); + assert_eq!(result.status, RunStatus::Ready); + assert_eq!( + result.data["resolved"], "https://examplepost.com/fusion-energy", + "{:?}", + result.diagnostics + ); + assert_eq!(result.data["publisher"], "examplepost.com"); + assert!(result.provenance.iter().any(|p| { + p.resolved_url.as_deref() == Some("https://examplepost.com/fusion-energy") + && p.publisher_host.as_deref() == Some("examplepost.com") + })); +} diff --git a/crates/toolbox/tests/templates.rs b/crates/toolbox/tests/templates.rs index f5856ad0..9141aa75 100644 --- a/crates/toolbox/tests/templates.rs +++ b/crates/toolbox/tests/templates.rs @@ -83,7 +83,10 @@ async fn every_template_matches_its_fixtures() { .filter(|s| s.starts_with("http")) { assert!( - result.provenance.iter().any(|p| p.url == url), + result + .provenance + .iter() + .any(|p| p.url == url || p.resolved_url.as_deref() == Some(url)), "{name}: {url} has no provenance" ); } @@ -120,3 +123,43 @@ async fn every_template_matches_its_fixtures() { } assert!(checked >= 12, "{checked} fixture cases"); } + +fn template(id: &str) -> octosense_toolbox::library::Template { + let (_, manifest, source) = BUILTIN + .iter() + .find(|(i, _, _)| *i == id) + .expect("template in BUILTIN"); + Template::from_parts(manifest, source, TemplateOrigin::Library).unwrap() +} + +/// The dossier template's own fixtures: its two cases (a clean two-question +/// run; a failed read and an empty question) replay green against the ported +/// template. +#[tokio::test] +async fn the_dossier_fixtures_replay() { + let app = AppContext::new("os.news", std::env::temp_dir()).grant("research"); + let template = template("dossier"); + for (_, mut case) in fixture_cases("dossier") { + let name = format!("dossier/{}", case.name); + let result = run( + &template, + &app, + case.params.clone(), + &fixture::host(&case.fixture), + RunOptions::default(), + ) + .await + .unwrap_or_else(|e| panic!("{name}: {e}")); + let expected = case + .expected + .take() + .unwrap_or_else(|| panic!("{name} has no expected result")); + assert_eq!( + result.status, expected.status, + "{name}: {:?}", + expected.reasons + ); + assert_eq!(result.status_reasons, expected.reasons, "{name}"); + assert_eq!(result.data, expected.data, "{name}"); + } +}