Search cached messages, on the socket and as tinyclient --text --search.

README.md

@@ -84,8 +84,9 @@ Signal identifies your own conversation as **Note to Self**, including messages
 sent from your phone. Existing cached self-conversations are relabeled when the
 account reconnects; their messages are preserved.
 
-Not built yet: message search (the switcher finds conversations by title in the local cache),
-notifications and a dock badge, video and screen share, voice messages, Signal attachments and
+Not built yet: message search in the window (the switcher finds conversations by title, and
+`tinyclient --text --search` searches the cache; the networks are not asked yet), notifications
+and a dock badge, video and screen share, voice messages, Signal attachments and
 safety numbers, Messages reactions and shown photos, Teams personal accounts. Each is a place to help; see [Contributing](#contributing).
 
 **Platforms.** CI builds and tests all three on every push (`.github/workflows/ci.yml`).

crates/chatd/src/server.rs

@@ -1413,6 +1413,17 @@ impl Chat {
                     Err(err) => internal(err),
                 }
             }
+            Request::SearchMessages { conversation, network, query, limit } => {
+                let query = query.trim();
+                let found = match query.is_empty() {
+                    true => Ok(Vec::new()),
+                    false => self.hub.store().search_messages(conversation.as_ref(), network, query, limit.min(PAGE_LIMIT)),
+                };
+                match found {
+                    Ok(hits) => Response::MessageHits { hits },
+                    Err(err) => internal(err),
+                }
+            }
             Request::SetDraft { conversation, text } => match self.hub.store().set_draft(&conversation, &text) {
                 Ok(true) => Response::Ok,
                 Ok(false) => error(ErrorCode::NotFound, "no such conversation"),

crates/chatd/src/store.rs

@@ -8,9 +8,9 @@
 
 use protocol::{
     clip, AccountKey, CallSupport, Conversation, ConversationKind, Folder, Group, InboxCounts,
-    InboxFilter, Key, Mark, MediaFile, MediaKind, Message, Network, Page, Person, Presence,
-    Reaction, Relation, SendState, SharedItem, SharedKind, Shelf, Space, Special, DRAFT_LIMIT,
-    FOLDERS_LIMIT, FOLDERS_PER_ROW, FOLDER_MEMBERS,
+    InboxFilter, Key, Mark, MediaFile, MediaKind, Message, MessageHit, Network, Page, Person,
+    Presence, Reaction, Relation, SendState, SharedItem, SharedKind, Shelf, Space, Special,
+    DRAFT_LIMIT, FOLDERS_LIMIT, FOLDERS_PER_ROW, FOLDER_MEMBERS,
 };
 use rusqlite::types::Value;
 use rusqlite::{params, params_from_iter, Connection, OptionalExtension, Row, Transaction};
@@ -503,6 +503,28 @@ fn like_prefix(query: &str) -> String {
     pattern
 }
 
+/// The text on one line, cut to a window that starts a little before the first match of
+/// `query`, at most 200 characters. Nothing marks the match here; the shell draws that.
+fn snippet(text: &str, query: &str) -> String {
+    let flat = text.split_whitespace().collect::<Vec<_>>().join(" ");
+    let needle = query.to_lowercase();
+    // A lowercased byte index can sit off a boundary or past the end by a character or two
+    // (some letters lowercase to more than one); walking back to one is close enough.
+    let mut from = flat.to_lowercase().find(&needle).map_or(0, |at| at.saturating_sub(60)).min(flat.len());
+    while from > 0 && !flat.is_char_boundary(from) {
+        from -= 1;
+    }
+    let cut: String = flat[from..].chars().take(200).collect();
+    let began = from > 0;
+    let ended = flat[from..].chars().count() > 200;
+    match (began, ended) {
+        (true, true) => format!("...{cut}..."),
+        (true, false) => format!("...{cut}"),
+        (false, true) => format!("{cut}..."),
+        (false, false) => cut,
+    }
+}
+
 /// The seven `media` columns of a left join, starting at `at`.
 fn file_at(row: &Row, at: usize) -> rusqlite::Result<Option<MediaFile>> {
     let Some(id) = row.get::<_, Option<String>>(at)? else {
@@ -1471,6 +1493,57 @@ impl Store {
             .collect()
     }
 
+    /// Cached messages whose text contains `query`, newest first. Hidden conversations and
+    /// message requests are not in it, and nor are messages that never went: both are kept out
+    /// of everything that counts, and a search does not get around that. The cache alone: what
+    /// a network itself finds lands when it answers, which is the shell's own later ask.
+    pub fn search_messages(
+        &self,
+        conversation: Option<&Key>,
+        network: Option<Network>,
+        query: &str,
+        limit: u32,
+    ) -> rusqlite::Result<Vec<MessageHit>> {
+        let pattern = format!("%{}%", like_prefix(query));
+        self.db
+            .prepare(
+                "SELECT m.network, m.account, m.conversation, m.id, m.sent, m.text, m.body, c.title
+                   FROM message m
+                   JOIN conversation c ON c.network = m.network AND c.account = m.account
+                      AND c.id = m.conversation
+                  WHERE m.pending = 0 AND m.text LIKE ?5 ESCAPE '\\'
+                    AND (?1 IS NULL OR m.network = ?1)
+                    AND (?2 IS NULL OR (m.network = ?2 AND m.account = ?3 AND m.conversation = ?4))
+                    AND NOT c.hidden AND c.special != 'request'
+                  ORDER BY m.sent DESC, length(m.id) DESC, m.id DESC LIMIT ?6",
+            )?
+            .query_map(
+                params![
+                    network.map(|network| word(&network)),
+                    conversation.map(|key| word(&key.network)),
+                    conversation.map(|key| key.account.as_str()),
+                    conversation.map(|key| key.id.as_str()),
+                    pattern,
+                    limit,
+                ],
+                |row| {
+                    let text: String = row.get(5)?;
+                    let body: String = row.get(6)?;
+                    let message: Message = serde_json::from_str(&body).map_err(broken)?;
+                    Ok(MessageHit {
+                        conversation: key_at(row)?,
+                        message_id: row.get(3)?,
+                        sent: row.get(4)?,
+                        sender: message.sender.name,
+                        snippet: snippet(&text, query),
+                        title: row.get(7)?,
+                        outgoing: message.outgoing,
+                    })
+                },
+            )?
+            .collect()
+    }
+
     /// What the user typed and did not send. False when there is no such conversation.
     pub fn set_draft(&mut self, conversation: &Key, draft: &str) -> rusqlite::Result<bool> {
         let mut draft = draft.to_owned();
@@ -2477,6 +2550,62 @@ mod tests {
         page.messages.iter().map(|message| message.id.as_str()).collect()
     }
 
+    #[test]
+    fn a_message_search_finds_the_words_said_newest_first_and_skips_what_never_went_or_went_hidden() {
+        let (mut store, key) = store();
+        let account = key.account();
+        let hidden = Key::new(&account, "secret");
+        store
+            .upsert_conversation(&account, &ConversationRow {
+                filing: Some(Filing { hidden: true, ..Filing::default() }),
+                ..ConversationRow::new("secret", ConversationKind::Direct, "Hidden")
+            })
+            .unwrap();
+        store.ingest(&key, &said("1", "hello world", 10), false).unwrap();
+        store.ingest(&key, &said("2", "WORLD peace", 20), true).unwrap();
+        let never = Message { state: SendState::Pending, ..said("3", "world in waiting", 30) };
+        store.ingest(&key, &never, false).unwrap();
+        store.ingest(&hidden, &said("4", "world of secrets", 40), true).unwrap();
+        let request = Key::new(&account, "stranger");
+        store
+            .upsert_conversation(&account, &ConversationRow {
+                special: Special::Request,
+                ..ConversationRow::new("stranger", ConversationKind::Direct, "Stranger")
+            })
+            .unwrap();
+        store.ingest(&request, &said("5", "world of tickets", 50), true).unwrap();
+
+        let hits = store.search_messages(None, None, "world", 10).unwrap();
+        assert_eq!(hits.iter().map(|hit| hit.message_id.as_str()).collect::<Vec<_>>(), ["2", "1"]);
+        assert_eq!((hits[0].sender.as_str(), hits[0].snippet.as_str()), ("mara", "WORLD peace"));
+        assert!(!hits[0].outgoing && hits[0].title == "#general");
+        assert_eq!(hits[0].conversation, key);
+
+        // Named conversation and network narrow; a hidden conversation answers nothing at all.
+        assert_eq!(store.search_messages(Some(&key), None, "world", 10).unwrap().len(), 2);
+        assert!(store.search_messages(Some(&hidden), None, "world", 10).unwrap().is_empty());
+        assert_eq!(store.search_messages(None, Some(Network::Slack), "world", 10).unwrap().len(), 0);
+        assert_eq!(store.search_messages(None, Some(Network::Discord), "world", 10).unwrap().len(), 2);
+
+        // Case goes for nothing, a longer cut carries the marker, and an empty query finds none.
+        let cut = store.search_messages(None, None, "peace", 1).unwrap();
+        assert_eq!(cut.len(), 1);
+        assert!(store.search_messages(None, None, "  ", 10).unwrap().is_empty());
+
+        // What the user typed means itself: a percent is not a wildcard.
+        store.ingest(&key, &said("5", "we matched 50% there", 50), false).unwrap();
+        assert_eq!(store.search_messages(None, None, "50%", 10).unwrap().len(), 1);
+        assert_eq!(store.search_messages(None, None, "0%_t", 10).unwrap().len(), 0);
+
+        // A match far into a long message is still the hit's window, marked on both ends.
+        let long = format!("{} the needle turns up here {}", "filler ".repeat(60), "and the text goes on".repeat(20));
+        store.ingest(&key, &said("6", &long, 60), false).unwrap();
+        let far = store.search_messages(None, None, "needle", 1).unwrap();
+        assert_eq!(far.len(), 1);
+        assert!(far[0].snippet.starts_with("...") && far[0].snippet.ends_with("..."), "{}", far[0].snippet);
+        assert!(far[0].snippet.contains("needle turns up"), "{}", far[0].snippet);
+    }
+
     #[test]
     fn pages_walk_a_conversation_both_ways_and_say_when_there_is_more() {
         let (mut store, key) = store();

crates/protocol/src/lib.rs

@@ -899,6 +899,26 @@ pub struct Watching {
     pub rows: Vec<Key>,
 }
 
+/// One hit of a message search: where the message is, and enough of it to choose by.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct MessageHit {
+    pub conversation: Key,
+    /// The message's id, to land on it with `Page::Around`.
+    pub message_id: String,
+    /// Unix seconds.
+    pub sent: i64,
+    /// Who sent it, as the message's own row says.
+    pub sender: String,
+    /// The text around the match, on one line, at most 200 characters.
+    pub snippet: String,
+    /// The conversation's title, so a result row says where it is without a second request.
+    #[serde(default, skip_serializing_if = "String::is_empty")]
+    pub title: String,
+    /// Mine, so the row can say "You".
+    #[serde(default, skip_serializing_if = "is_false")]
+    pub outgoing: bool,
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(tag = "op", rename_all = "snake_case")]
 pub enum Request {
@@ -960,6 +980,18 @@ pub enum Request {
         query: String,
         limit: u32,
     },
+    /// Cached messages whose text contains `query`, newest first. `conversation` narrows to one
+    /// conversation, `network` to one network. Hidden conversations and message requests stay
+    /// out of it, as they stay out of everything that counts. From the cache alone: what a
+    /// network itself finds lands as it answers, and is a later step.
+    SearchMessages {
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        conversation: Option<Key>,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        network: Option<Network>,
+        query: String,
+        limit: u32,
+    },
     /// Keeps what the user typed and did not send. Empty text forgets it. Local: no network
     /// hears of it and no change is announced.
     SetDraft {
@@ -1476,6 +1508,10 @@ pub enum Response {
         /// The cache holds messages newer than the last one here.
         newer: bool,
     },
+    /// Answers `SearchMessages`: the newest hits first, at most the limit asked for.
+    MessageHits {
+        hits: Vec<MessageHit>,
+    },
     Sent {
         message: Message,
     },
@@ -1687,6 +1723,38 @@ mod tests {
         assert_eq!(serde_json::from_str::<Outgoing>(&json).unwrap(), quiet);
     }
 
+    #[test]
+    fn a_message_search_and_its_hits_read_off_the_wire() {
+        let ask: Request = serde_json::from_str(r#"{"op":"search_messages","query":"lichen","limit":20}"#).unwrap();
+        let Request::SearchMessages { conversation, network, query, limit } = &ask else {
+            panic!("not a search")
+        };
+        assert!(conversation.is_none() && network.is_none() && query == "lichen" && *limit == 20);
+        let with_filters = Request::SearchMessages {
+            conversation: Some(Key { network: Network::Slack, account: "T1".into(), id: "C1".into() }),
+            network: Some(Network::Slack),
+            query: "lichen".into(),
+            limit: 20,
+        };
+        assert_eq!(
+            serde_json::to_string(&with_filters).unwrap(),
+            r#"{"op":"search_messages","conversation":{"network":"slack","account":"T1","id":"C1"},"network":"slack","query":"lichen","limit":20}"#
+        );
+        let hit = MessageHit {
+            conversation: Key { network: Network::Discord, account: "1".into(), id: "C2".into() },
+            message_id: "9".into(),
+            sent: 20,
+            sender: "mara".into(),
+            snippet: "hello world".into(),
+            title: String::new(),
+            outgoing: false,
+        };
+        assert_eq!(
+            serde_json::to_string(&Response::MessageHits { hits: vec![hit] }).unwrap(),
+            r#"{"op":"message_hits","hits":[{"conversation":{"network":"discord","account":"1","id":"C2"},"message_id":"9","sent":20,"sender":"mara","snippet":"hello world"}]}"#
+        );
+    }
+
     #[test]
     fn a_frame_too_large_to_read_is_never_written() {
         let mut out = Vec::new();

crates/tinyclient/src/main.rs

@@ -59,6 +59,26 @@ fn print_status() -> io::Result<()> {
     Ok(())
 }
 
+/// The shell without a window: what a search of the cache finds, as lines.
+fn print_search(query: &str) -> io::Result<()> {
+    let request = Request::SearchMessages { conversation: None, network: None, query: query.to_owned(), limit: PAGE_LIMIT };
+    let hits = match ask(&request)? {
+        Response::MessageHits { hits } => hits,
+        other => return Err(unexpected(other)),
+    };
+    let now = view::now();
+    for hit in hits {
+        println!(
+            "{:<8} {:<24} {:>4}  {}",
+            hit.conversation.network.label(),
+            hit.title,
+            view::age(now, hit.sent),
+            hit.snippet
+        );
+    }
+    Ok(())
+}
+
 /// A windowed program starts with no console, so what it prints goes nowhere. When output is not
 /// already a pipe or a file, write to the console of the program that started it, if any.
 #[cfg(all(windows, feature = "gui"))]
@@ -86,22 +106,37 @@ fn borrow_console() {
 }
 
 fn main() -> ExitCode {
+    let args: Vec<String> = std::env::args().skip(1).collect();
+    let query = match args.iter().position(|arg| arg == "--search") {
+        Some(at) => match args.get(at + 1) {
+            Some(query) => Some(query.as_str()),
+            None => {
+                eprintln!("tinyclient: --search needs what to look for");
+                return ExitCode::from(64);
+            }
+        },
+        None => None,
+    };
     #[cfg(all(windows, feature = "gui"))]
-    if std::env::args().any(|arg| arg == "--version" || arg == "--text") {
+    if args.iter().any(|arg| arg == "--version" || arg == "--text") || query.is_some() {
         borrow_console();
     }
     // Before the window opens: it is how an installed package proves it runs, with no display.
-    if std::env::args().nth(1).as_deref() == Some("--version") {
+    if args.first().map(String::as_str) == Some("--version") {
         println!("tinyclient {}", env!("CARGO_PKG_VERSION"));
         return ExitCode::SUCCESS;
     }
     #[cfg(feature = "gui")]
-    if !std::env::args().any(|arg| arg == "--text") {
+    if !args.iter().any(|arg| arg == "--text") && query.is_none() {
         gui::run();
         return ExitCode::SUCCESS;
     }
 
-    match print_status() {
+    let done = match query {
+        Some(query) => print_search(query),
+        None => print_status(),
+    };
+    match done {
         Ok(()) => ExitCode::SUCCESS,
         Err(err) if matches!(err.kind(), ErrorKind::NotFound | ErrorKind::ConnectionRefused) => {
             eprintln!("chatd is not running at {}", chat_sock_path().display());

docs/architecture.md

@@ -37,6 +37,7 @@ How the processes fit together and who may edit what. The contract itself is the
 | `Threads {space, offset, limit}`, `Topics {forum, offset, limit}` | `Conversations` | store; `Command::OpenThreads` at offset 0 |
 | `InVoice {network}`, `Around {network}` | `Conversations`; `Around {requests, in_call, online, recent}` | hub (occupants, presence, activity) and store |
 | `Find {network, query, limit}` | `Conversations` (title contains, newest first, cache only) | store |
+| `SearchMessages {conversation, network, query, limit}` | `MessageHits {hits (where, snippet, sender, title)}` (text contains, newest first, cache only) | store |
 | `Spaces {network}`, `Tree {space, offset, limit, peek}` | `Spaces`; `Tree {groups (kind, count), conversations, total}` | store; `Command::OpenSpace` |
 | `Conversation`, `Collapse`, `Star`, `SetDraft` | `Conversation`; `Ok` | store (local only) |
 | `People {network, relation, query, offset, limit}` | `People {people, total}` | store; `Command::SearchPeople`, debounced |