Fix import of old WhatsApp msgstore.db backups (#272)

wesm · claude · web-flow · commit a865f823431e · 2026-04-15T08:49:13.000-05:00
## Summary - Detect missing `chat.group_type` and `message_media.media_caption` columns in old WhatsApp schemas (pre-2022) via `PRAGMA table_info`, defaulting to `0`/`NULL` when absent - Column cache scoped per `(db pointer, table)` so multiple databases with different schemas in the same process get independent detection Fixes #271 --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
diff --git a/internal/whatsapp/queries.go b/internal/whatsapp/queries.go
@@ -4,27 +4,87 @@ import (
 	"database/sql"
 	"fmt"
 	"strings"
+	"sync"
 )
 
+// hasColumn checks whether a table has a given column using PRAGMA table_info.
+// Results are cached per (db pointer, table) to avoid repeated PRAGMA queries
+// while correctly handling multiple databases with different schemas.
+var (
+	columnCache   = make(map[columnCacheKey]map[string]bool)
+	columnCacheMu sync.Mutex
+)
+
+type columnCacheKey struct {
+	db    *sql.DB
+	table string
+}
+
+func hasColumn(db *sql.DB, table, column string) bool {
+	columnCacheMu.Lock()
+	defer columnCacheMu.Unlock()
+
+	key := columnCacheKey{db: db, table: table}
+	cols, ok := columnCache[key]
+	if !ok {
+		cols = make(map[string]bool)
+		rows, err := db.Query(
+			fmt.Sprintf("PRAGMA table_info(%s)", table),
+		)
+		if err == nil {
+			defer func() { _ = rows.Close() }()
+			for rows.Next() {
+				var cid int
+				var name, colType string
+				var notNull, pk int
+				var dfltValue sql.NullString
+				if err := rows.Scan(
+					&cid, &name, &colType,
+					&notNull, &dfltValue, &pk,
+				); err == nil {
+					cols[name] = true
+				}
+			}
+		}
+		columnCache[key] = cols
+	}
+	return cols[column]
+}
+
+// resetColumnCache clears the cached column info (for testing).
+func resetColumnCache() {
+	columnCacheMu.Lock()
+	defer columnCacheMu.Unlock()
+	columnCache = make(map[columnCacheKey]map[string]bool)
+}
+
 // fetchChats returns all non-hidden chats from the WhatsApp database.
 // Joins with the jid table to get JID details for each chat.
+// Handles old WhatsApp schemas that lack the group_type column.
 func fetchChats(db *sql.DB) ([]waChat, error) {
-	rows, err := db.Query(`
+	hasGroupType := hasColumn(db, "chat", "group_type")
+
+	groupTypeExpr := "0"
+	if hasGroupType {
+		groupTypeExpr = "COALESCE(c.group_type, 0)"
+	}
+
+	rows, err := db.Query(fmt.Sprintf(`
 		SELECT
 			c._id,
 			c.jid_row_id,
 			j.raw_string,
 			COALESCE(j.user, ''),
 			COALESCE(j.server, ''),
 			c.subject,
-			COALESCE(c.group_type, 0),
+			%s,
 			COALESCE(c.hidden, 0),
 			COALESCE(c.sort_timestamp, 0)
 		FROM chat c
 		JOIN jid j ON c.jid_row_id = j._id
 		WHERE COALESCE(c.hidden, 0) = 0
 		ORDER BY c.sort_timestamp DESC
-	`)
+	`, groupTypeExpr))
 	if err != nil {
 		return nil, fmt.Errorf("fetch chats: %w", err)
 	}
@@ -94,11 +154,14 @@ func fetchMessages(db *sql.DB, chatRowID int64, afterID int64, limit int) ([]waM
 
 // fetchMedia returns media metadata for a batch of message row IDs.
 // Returns a map of message_row_id → waMedia.
+// Handles old WhatsApp schemas that lack the media_caption column.
 func fetchMedia(db *sql.DB, messageRowIDs []int64) (map[int64]waMedia, error) {
 	if len(messageRowIDs) == 0 {
 		return make(map[int64]waMedia), nil
 	}
 
+	hasCaption := hasColumn(db, "message_media", "media_caption")
+
 	result := make(map[int64]waMedia)
 
 	// Process in chunks to stay within SQLite's parameter limit.
@@ -117,19 +180,24 @@ func fetchMedia(db *sql.DB, messageRowIDs []int64) (map[int64]waMedia, error) {
 			args[j] = id
 		}
 
+		captionExpr := "NULL"
+		if hasCaption {
+			captionExpr = "mm.media_caption"
+		}
+
 		query := fmt.Sprintf(`
 			SELECT
 				mm.message_row_id,
 				mm.mime_type,
-				mm.media_caption,
+				%s,
 				mm.file_size,
 				mm.file_path,
 				mm.width,
 				mm.height,
 				mm.media_duration
 			FROM message_media mm
 			WHERE mm.message_row_id IN (%s)
-		`, strings.Join(placeholders, ","))
+		`, captionExpr, strings.Join(placeholders, ","))
 
 		rows, err := db.Query(query, args...)
 		if err != nil {
diff --git a/internal/whatsapp/queries_test.go b/internal/whatsapp/queries_test.go
@@ -7,6 +7,236 @@ import (
 	_ "github.com/mattn/go-sqlite3"
 )
 
+func TestFetchChatsOldSchema(t *testing.T) {
+	// Old WhatsApp schemas (pre-2022) lack the group_type column on chat.
+	// fetchChats should handle this gracefully, defaulting group_type to 0.
+	db, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = db.Close() }()
+	resetColumnCache()
+
+	// Create old-style chat table without group_type.
+	_, err = db.Exec(`
+		CREATE TABLE jid (
+			_id INTEGER PRIMARY KEY,
+			user TEXT,
+			server TEXT,
+			raw_string TEXT
+		);
+		CREATE TABLE chat (
+			_id INTEGER PRIMARY KEY,
+			jid_row_id INTEGER UNIQUE,
+			hidden INTEGER,
+			subject TEXT,
+			sort_timestamp INTEGER
+		);
+
+		INSERT INTO jid (_id, user, server, raw_string)
+			VALUES (1, '447700900000', 's.whatsapp.net', '447700900000@s.whatsapp.net');
+		INSERT INTO jid (_id, user, server, raw_string)
+			VALUES (2, '120363001234567890', 'g.us', '120363001234567890@g.us');
+
+		INSERT INTO chat (_id, jid_row_id, hidden, subject, sort_timestamp)
+			VALUES (10, 1, 0, NULL, 1609459200000);
+		INSERT INTO chat (_id, jid_row_id, hidden, subject, sort_timestamp)
+			VALUES (20, 2, 0, 'Family Group', 1609459300000);
+	`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	chats, err := fetchChats(db)
+	if err != nil {
+		t.Fatalf("fetchChats with old schema: %v", err)
+	}
+
+	if len(chats) != 2 {
+		t.Fatalf("expected 2 chats, got %d", len(chats))
+	}
+
+	// All chats should have GroupType=0 since column is missing.
+	for _, c := range chats {
+		if c.GroupType != 0 {
+			t.Errorf("chat %d: GroupType = %d, want 0", c.RowID, c.GroupType)
+		}
+	}
+
+	// Group chat (g.us) should still be detected via server.
+	group := chats[0] // sorted by sort_timestamp DESC
+	if group.Server != "g.us" {
+		t.Errorf("expected first chat to be group (g.us), got server=%q", group.Server)
+	}
+	if !isGroupChat(group) {
+		t.Error("g.us chat should be detected as group even without group_type column")
+	}
+}
+
+func TestFetchChatsNewSchema(t *testing.T) {
+	// New WhatsApp schemas have group_type on chat.
+	db, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = db.Close() }()
+	resetColumnCache()
+
+	_, err = db.Exec(`
+		CREATE TABLE jid (
+			_id INTEGER PRIMARY KEY,
+			user TEXT,
+			server TEXT,
+			raw_string TEXT
+		);
+		CREATE TABLE chat (
+			_id INTEGER PRIMARY KEY,
+			jid_row_id INTEGER UNIQUE,
+			hidden INTEGER,
+			subject TEXT,
+			sort_timestamp INTEGER,
+			group_type INTEGER
+		);
+
+		INSERT INTO jid (_id, user, server, raw_string)
+			VALUES (1, '120363009999', 'g.us', '120363009999@g.us');
+		INSERT INTO chat (_id, jid_row_id, hidden, subject, sort_timestamp, group_type)
+			VALUES (10, 1, 0, 'Work Chat', 1609459200000, 1);
+	`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	chats, err := fetchChats(db)
+	if err != nil {
+		t.Fatalf("fetchChats with new schema: %v", err)
+	}
+
+	if len(chats) != 1 {
+		t.Fatalf("expected 1 chat, got %d", len(chats))
+	}
+	if chats[0].GroupType != 1 {
+		t.Errorf("GroupType = %d, want 1", chats[0].GroupType)
+	}
+}
+
+func TestFetchMediaOldSchema(t *testing.T) {
+	// Old WhatsApp schemas lack media_caption on message_media.
+	db, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = db.Close() }()
+	resetColumnCache()
+
+	_, err = db.Exec(`
+		CREATE TABLE message_media (
+			message_row_id INTEGER PRIMARY KEY,
+			mime_type TEXT,
+			file_size INTEGER,
+			file_path TEXT,
+			width INTEGER,
+			height INTEGER,
+			media_duration INTEGER
+		);
+
+		INSERT INTO message_media (message_row_id, mime_type, file_size, file_path, width, height, media_duration)
+			VALUES (100, 'image/jpeg', 54321, 'Media/IMG-20200101.jpg', 1920, 1080, 0);
+	`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mediaMap, err := fetchMedia(db, []int64{100})
+	if err != nil {
+		t.Fatalf("fetchMedia with old schema: %v", err)
+	}
+
+	m, ok := mediaMap[100]
+	if !ok {
+		t.Fatal("expected media for message 100")
+	}
+	if m.MediaCaption.Valid {
+		t.Error("MediaCaption should be NULL for old schema")
+	}
+	if !m.MimeType.Valid || m.MimeType.String != "image/jpeg" {
+		t.Errorf("MimeType = %v, want image/jpeg", m.MimeType)
+	}
+}
+
+func TestColumnCacheScopedPerDB(t *testing.T) {
+	// Verify that inspecting an old-schema DB then a new-schema DB
+	// (and vice versa) produces correct results without resetColumnCache.
+	resetColumnCache()
+
+	// DB 1: old schema, no group_type.
+	oldDB, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = oldDB.Close() }()
+	_, err = oldDB.Exec(`
+		CREATE TABLE jid (_id INTEGER PRIMARY KEY, user TEXT, server TEXT, raw_string TEXT);
+		CREATE TABLE chat (_id INTEGER PRIMARY KEY, jid_row_id INTEGER UNIQUE, hidden INTEGER, subject TEXT, sort_timestamp INTEGER);
+		INSERT INTO jid VALUES (1, '441234567890', 's.whatsapp.net', '441234567890@s.whatsapp.net');
+		INSERT INTO chat VALUES (1, 1, 0, NULL, 1000);
+	`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// DB 2: new schema, has group_type.
+	newDB, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = newDB.Close() }()
+	_, err = newDB.Exec(`
+		CREATE TABLE jid (_id INTEGER PRIMARY KEY, user TEXT, server TEXT, raw_string TEXT);
+		CREATE TABLE chat (_id INTEGER PRIMARY KEY, jid_row_id INTEGER UNIQUE, hidden INTEGER, subject TEXT, sort_timestamp INTEGER, group_type INTEGER);
+		INSERT INTO jid VALUES (1, '120363009999', 'g.us', '120363009999@g.us');
+		INSERT INTO chat VALUES (1, 1, 0, 'Test Group', 2000, 3);
+	`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Query old DB first — should NOT cache "no group_type" for new DB.
+	oldChats, err := fetchChats(oldDB)
+	if err != nil {
+		t.Fatalf("old DB: %v", err)
+	}
+	if oldChats[0].GroupType != 0 {
+		t.Errorf("old DB: GroupType = %d, want 0", oldChats[0].GroupType)
+	}
+
+	// Query new DB — must see group_type despite old DB being queried first.
+	newChats, err := fetchChats(newDB)
+	if err != nil {
+		t.Fatalf("new DB: %v", err)
+	}
+	if newChats[0].GroupType != 3 {
+		t.Errorf("new DB: GroupType = %d, want 3", newChats[0].GroupType)
+	}
+
+	// Reverse: query new DB again then old DB again — still correct.
+	newChats2, err := fetchChats(newDB)
+	if err != nil {
+		t.Fatalf("new DB (2nd): %v", err)
+	}
+	if newChats2[0].GroupType != 3 {
+		t.Errorf("new DB (2nd): GroupType = %d, want 3", newChats2[0].GroupType)
+	}
+
+	oldChats2, err := fetchChats(oldDB)
+	if err != nil {
+		t.Fatalf("old DB (2nd): %v", err)
+	}
+	if oldChats2[0].GroupType != 0 {
+		t.Errorf("old DB (2nd): GroupType = %d, want 0", oldChats2[0].GroupType)
+	}
+}
+
 func TestFetchLidMap(t *testing.T) {
 	db, err := sql.Open("sqlite3", ":memory:")
 	if err != nil {
