toolbox: the dossier template; search categories; resolved_url and publisher_host

crates/toolbox/src/fixture.rs

@@ -88,6 +88,10 @@ pub struct RecordedPage {
     pub title: Option<String>,
     #[serde(default)]
     pub delay_ms: u64,
+    /// Where the read ended after redirects (a Google News link's publisher
+    /// page); absent means the backend did not say.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub final_url: Option<String>,
     /// Replay a failed read (blocked, no main text, …).
     #[serde(default)]
     pub error: Option<String>,
@@ -195,6 +199,7 @@ impl ResearchBackend for FixtureBackend {
             Ok(PageText {
                 text: page.text.clone(),
                 title: page.title.clone(),
+                final_url: page.final_url.clone(),
             })
         })
     }

crates/toolbox/src/host.rs

@@ -139,6 +139,15 @@ pub struct Provenance {
     /// SHA-256 of the evidence text the host kept, once read.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub evidence_sha256: Option<String>,
+    /// Where the read of it ended, after redirects (a Google News link's
+    /// publisher page); absent when not read or still on the aggregator.
+    /// `url` stays the search result's link.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub resolved_url: Option<String>,
+    /// The publisher's host (lowercase, no `www.`): `resolved_url`'s, else
+    /// `url`'s unless that is an aggregator's.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub publisher_host: Option<String>,
     /// The provider or method that produced it (`search:gdelt`, `article`).
     pub via: String,
 }

crates/toolbox/src/library.rs

@@ -146,6 +146,7 @@ pub static BUILTIN: &[(&str, &str, &str)] = builtin!(
     "market-brief",
     "briefing",
     "compare",
+    "dossier",
 );
 
 const BUILTIN_LOCK: &str = include_str!("../templates/library.lock.json");

crates/toolbox/src/modules.rs

@@ -111,15 +111,16 @@ pub const RESEARCH: ModuleSpec = ModuleSpec {
         MethodSpec {
             name: "search",
             kind: MethodKind::Search,
-            description: "One query through the provider chain; structured items with host-assigned ids, readable items first (`readable: false` marks one the backend cannot read)",
+            description: "One query through the provider chain (the metasearch `category`, `news` by default, must be in the app's grant; `max_age_hours` is clamped to its `max_age_days`); structured items with host-assigned ids, readable items first (`readable: false` marks one the backend cannot read)",
             input: || {
                 json!({"type": "object", "additionalProperties": false, "required": ["topic"],
                     "properties": {
                         "topic": string(1, 160),
                         "language": string(2, 16),
                         "region": string(2, 8),
                         "limit": {"type": "integer", "minimum": 1, "maximum": 10},
-                        "max_age_hours": {"type": "integer", "minimum": 1, "maximum": 720}
+                        "max_age_hours": {"type": "integer", "minimum": 1, "maximum": 36000},
+                        "category": {"type": "string", "enum": ["news", "general", "it", "social", "science"]}
                     }})
             },
             output: || {
@@ -141,19 +142,21 @@ pub const RESEARCH: ModuleSpec = ModuleSpec {
         MethodSpec {
             name: "article",
             kind: MethodKind::Read,
-            description: "Read one search result's page and keep its main text as evidence; `on_topic: false` when the page does not mention the terms of the search that found it (the host will not digest it)",
+            description: "Read one search result's page and keep its main text as evidence; `resolved_url` is where the read ended (a Google News link's publisher page, null while unresolved) and `publisher_host` its host; `on_topic: false` when the page does not mention the terms of the search that found it (the host will not digest it)",
             input: || {
                 json!({"type": "object", "additionalProperties": false, "required": ["id"],
                     "properties": {"id": string(1, 64)}})
             },
             output: || {
                 json!({"type": "object", "additionalProperties": false,
-                    "required": ["id", "title", "url", "source", "language", "published_at",
-                        "excerpt", "chars", "truncated", "evidence_sha256", "on_topic"],
+                    "required": ["id", "title", "url", "resolved_url", "publisher_host", "source", "language",
+                        "published_at", "excerpt", "chars", "truncated", "evidence_sha256", "on_topic"],
                     "properties": {
                         "id": string(1, 64),
                         "title": string(0, 400),
                         "url": string(1, 2048),
+                        "resolved_url": {"description": "Where the read ended after redirects (a Google News link's publisher page); null when unknown or still on the aggregator"},
+                        "publisher_host": {"description": "The publisher's host (resolved_url's, else url's unless an aggregator's), or null"},
                         "source": string(0, 200),
                         "language": string(0, 16),
                         "published_at": string(0, 40),

crates/toolbox/src/peer.rs

@@ -174,11 +174,12 @@ pub fn catalog(library: &Library) -> Vec<Value> {
         decls.push(decl(
             SEARCH,
             "read",
-            "Search free news sources (Google News, GDELT, publisher feeds) within this app's granted languages, regions, domains and recency; returns dated items with their sources, saved in this app's toolbox folder.".into(),
+            "Search free sources through the octos metasearch within this app's granted languages, regions, domains, categories and recency: category `news` (default: Google News, GDELT, publisher feeds, Hacker News), `general` (general web search: DuckDuckGo, Bing, Brave, Google, Wikipedia), `it` (GitHub, Hacker News, Stack Exchange), `science` (arXiv, OpenAlex), `social` (Mastodon; posts are not returned as evidence). Returns dated items with their sources, saved in this app's toolbox folder.".into(),
             json!({
                 "type": "object", "required": ["query"], "additionalProperties": false,
                 "properties": {
                     "query": {"type": "string", "minLength": 1, "maxLength": 300},
+                    "category": {"type": "string", "enum": ["news", "general", "it", "science", "social"], "description": "metasearch category (default news; must be granted to the app)"},
                     "lang": {"type": "string", "description": "BCP-47 language to search in (must be granted to the app)"},
                     "region": {"type": "string", "description": "ISO 3166-1 alpha-2 region"},
                     "max_age_days": {"type": "integer", "minimum": 1, "description": "only material this recent"},
@@ -414,8 +415,21 @@ impl PeerToolbox {
         let count = args["count"]
             .as_u64()
             .map_or(DEFAULT_SEARCH_COUNT, |c| c.clamp(1, 30) as u32);
-        let narrowed = scope::narrow_search(&app.scope, query, lang, region, max_age_hours, count)
-            .map_err(|e| PeerError::new("denied", e))?;
+        let category = args["category"]
+            .as_str()
+            .map(str::trim)
+            .filter(|c| !c.is_empty())
+            .map(str::to_owned);
+        let narrowed = scope::narrow_search_in(
+            &app.scope,
+            category.as_deref().unwrap_or(scope::SEARCH_CATEGORY),
+            query,
+            lang,
+            region,
+            max_age_hours,
+            count,
+        )
+        .map_err(|e| PeerError::new("denied", e))?;
         let ctx = self.context(app, SEARCH, 0);
         let found = self
             .backend
@@ -428,6 +442,7 @@ impl PeerToolbox {
                     limit: narrowed.limit,
                     max_age_hours: narrowed.max_age_hours,
                     max_fetches: MAX_SEARCH_FETCHES,
+                    category: category.clone(),
                 },
             )
             .await
@@ -448,7 +463,7 @@ impl PeerToolbox {
             .collect();
         let now = self.stamp();
         let doc = json!({
-            "kind": "search", "app_id": app.app_id, "query": query, "retrieved_at": now,
+            "kind": "search", "app_id": app.app_id, "query": query, "category": category.as_deref().unwrap_or(scope::SEARCH_CATEGORY), "retrieved_at": now,
             "items": items, "providers": found.providers, "partial": found.partial, "notes": notes,
         });
         let file = save(&app.folder, "search", query, &doc)?;

crates/toolbox/src/research/live.rs

@@ -1212,6 +1212,7 @@ impl ResearchBackend for InterimResearch {
             Ok(PageText {
                 text,
                 title: Some(article.title.to_string()).filter(|t| !t.is_empty()),
+                final_url: Some(final_url),
             })
         })
     }
@@ -1397,6 +1398,7 @@ mod tests {
             limit: 5,
             max_age_hours: Some(72),
             max_fetches: 8,
+            category: None,
         }
     }
 

crates/toolbox/src/research/mod.rs

@@ -72,6 +72,10 @@ pub struct SearchQuery {
     /// Feeds and API responses the backend may fetch for this search, at
     /// most [`MAX_SEARCH_FETCHES`].
     pub max_fetches: u32,
+    /// The metasearch category, when not the backend's default (`news`):
+    /// `general`, `it`, `social`, `science`. Already within the grant.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub category: Option<String>,
 }
 
 /// One item a backend found. `via` names the provider.
@@ -136,6 +140,11 @@ pub struct PageText {
     pub text: String,
     #[serde(default)]
     pub title: Option<String>,
+    /// Where the read ended, after HTTP and browser (JavaScript) redirects:
+    /// for a Google News link, the publisher's page. `None` when the backend
+    /// cannot tell.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub final_url: Option<String>,
 }
 
 /// Finds and reads sources. The fixture backend, the interim live adapter,
@@ -315,11 +324,17 @@ impl ResearchHost {
         let region = input["region"].as_str().map(str::to_owned);
         let limit = input["limit"].as_u64().unwrap_or(5).clamp(1, 10) as u32;
         let requested_age = input["max_age_hours"].as_u64().map(|h| h as u32);
+        let category = input["category"]
+            .as_str()
+            .map(str::trim)
+            .filter(|c| !c.is_empty())
+            .map(str::to_owned);
         // octos's `Scope::search_args` (or the thin parser's same rules):
         // a language, region or category outside the grant is refused,
         // recency and the result count are clamped.
-        let narrowed = scope::narrow_search(
+        let narrowed = scope::narrow_search_in(
             scope,
+            category.as_deref().unwrap_or(scope::SEARCH_CATEGORY),
             &topic,
             language.as_deref(),
             region.as_deref(),
@@ -335,6 +350,7 @@ impl ResearchHost {
             limit,
             max_age_hours: narrowed.max_age_hours,
             max_fetches: MAX_SEARCH_FETCHES,
+            category,
         };
         let topic = Topic::new(&query.topic);
         let mut results = self.backend.search(ctx, query.clone()).await?;
@@ -380,6 +396,8 @@ impl ResearchHost {
                 published_at: item.published_at.clone(),
                 retrieved_at: queried_at.clone(),
                 evidence_sha256: None,
+                resolved_url: None,
+                publisher_host: None,
                 via: format!("search:{}", item.via),
             });
             items.push(json!({
@@ -504,10 +522,13 @@ impl ResearchHost {
         } else {
             item.title.clone()
         };
+        let (resolved_url, publisher_host) = resolve(&item.url, page.final_url.as_deref());
         let output = json!({
             "id": id,
             "title": clip(&title, 400),
             "url": item.url,
+            "resolved_url": resolved_url,
+            "publisher_host": publisher_host,
             "source": clip(&item.source, 200),
             "language": clip(&item.language, 16),
             "published_at": clip(&item.published_at, 40),
@@ -526,6 +547,8 @@ impl ResearchHost {
             published_at: item.published_at.clone(),
             retrieved_at,
             evidence_sha256: Some(hash),
+            resolved_url,
+            publisher_host,
             via: "article".into(),
         }];
         self.state(&ctx.run_id, |s| {
@@ -1000,6 +1023,26 @@ fn host_of(url: &str) -> Option<String> {
     (!host.is_empty()).then_some(host)
 }
 
+/// Where an item's read ended (`resolved_url`) and its publisher's host.
+/// A read that ended on an aggregator (still a Google News link) is not
+/// resolved; the publisher host then comes from the item's own link unless
+/// that is an aggregator's too.
+fn resolve(url: &str, final_url: Option<&str>) -> (Option<String>, Option<String>) {
+    let not_aggregator =
+        |u: &&str| host_of(u).is_some_and(|h| !AGGREGATOR_HOSTS.contains(&h.as_str()));
+    let resolved = final_url
+        .map(str::trim)
+        .filter(|u| u.starts_with("http://") || u.starts_with("https://"))
+        .filter(not_aggregator)
+        .filter(|u| u.len() <= 2048)
+        .map(str::to_owned);
+    let host = resolved
+        .as_deref()
+        .or(Some(url).filter(not_aggregator))
+        .and_then(host_of);
+    (resolved, host)
+}
+
 /// What identifies an item's publisher before it is read: its source name,
 /// and its link's host unless that is an aggregator's.
 fn publisher_keys(item: &FoundItem) -> Vec<String> {
@@ -1018,6 +1061,24 @@ fn publisher_keys(item: &FoundItem) -> Vec<String> {
 mod tests {
     use super::*;
 
+    #[test]
+    fn google_news_links_resolve_only_off_the_aggregator() {
+        let gn = "https://news.google.com/rss/articles/CBMi?oc=5";
+        assert_eq!(resolve(gn, None), (None, None));
+        assert_eq!(resolve(gn, Some(gn)), (None, None));
+        assert_eq!(
+            resolve(gn, Some("https://www.startupfortune.com/dhh/")),
+            (
+                Some("https://www.startupfortune.com/dhh/".into()),
+                Some("startupfortune.com".into())
+            )
+        );
+        assert_eq!(
+            resolve("https://bbc.co.uk/a", None),
+            (None, Some("bbc.co.uk".into()))
+        );
+    }
+
     #[test]
     fn refusals_mark_the_publisher_and_the_host_the_read_ended_on() {
         let bot = "bot_challenge: a DataDome challenge (not bypassed) (final URL: https://www.nytimes.com/a/b.html?x=1)";

crates/toolbox/src/research/octos.rs

@@ -433,7 +433,13 @@ impl ResearchBackend for OctosResearch {
             let now = chrono::Utc::now();
             let scope = &ctx.app.scope;
             let scoped = scope
-                .search_args(&search_args(&query, &self.config.category), now)
+                .search_args(
+                    &search_args(
+                        &query,
+                        query.category.as_deref().unwrap_or(&self.config.category),
+                    ),
+                    now,
+                )
                 .map_err(HostError::Denied)?;
             let request = search_request(&scoped, &self.config, now);
             let response = self.metasearch.search(&request).await;
@@ -527,6 +533,7 @@ impl ResearchBackend for OctosResearch {
             Ok(PageText {
                 text: page.text,
                 title: page.meta.title.filter(|t| !t.trim().is_empty()),
+                final_url: Some(page.final_url),
             })
         })
     }
@@ -545,11 +552,12 @@ impl ResearchBackend for OctosResearch {
                 .map_err(HostError::Denied)?;
             let links = extract_links(&page.html, &page.final_url);
             Ok(LinkedPage {
-                final_url: page.final_url,
                 page: PageText {
                     text: page.text,
                     title: page.meta.title.filter(|t| !t.trim().is_empty()),
+                    final_url: Some(page.final_url.clone()),
                 },
+                final_url: page.final_url,
                 links,
             })
         })
@@ -702,6 +710,7 @@ mod tests {
             limit: 5,
             max_age_hours: Some(72),
             max_fetches: 8,
+            category: None,
         };
         let now = chrono::Utc::now();
         let scoped = scope
@@ -842,6 +851,7 @@ mod tests {
             limit: 5,
             max_age_hours: None,
             max_fetches: 8,
+            category: None,
         };
         let now = chrono::Utc::now();
         let scoped = grant

crates/toolbox/src/runner.rs

@@ -533,7 +533,13 @@ pub async fn run<H: ToolboxHost + ?Sized>(
         provenance_order
             .iter()
             .filter_map(|id| provenance.get(id))
-            .filter(|p| referenced.contains(p.id.as_str()) || referenced.contains(p.url.as_str()))
+            .filter(|p| {
+                referenced.contains(p.id.as_str())
+                    || referenced.contains(p.url.as_str())
+                    || p.resolved_url
+                        .as_deref()
+                        .is_some_and(|u| referenced.contains(u))
+            })
             .cloned()
             .collect()
     } else {
@@ -642,7 +648,10 @@ fn finish_output(
     schema
         .validate(&data)
         .map_err(|e| format!("output schema: {e}"))?;
-    let urls: BTreeSet<&str> = provenance.values().map(|p| p.url.as_str()).collect();
+    let urls: BTreeSet<&str> = provenance
+        .values()
+        .flat_map(|p| std::iter::once(p.url.as_str()).chain(p.resolved_url.as_deref()))
+        .collect();
     if let Some(bad) = json::strings(&data)
         .into_iter()
         .find(|s| json::contains_url(s) && !urls.contains(s))