Database picker: search on the server instead of listing every database

CHANGELOG.md

@@ -4,7 +4,8 @@
 
 - Read query results as Arrow directly from `POST /v1/query`, so a fast query takes one request instead of two. Async and too-large results still come from the Results API.
 - Show the real error for failed queries: read `error_message` from query runs and failed results, and show the `message` and `code` from Hotdata error bodies instead of raw JSON.
-- Follow pagination on `/v1/databases` and `/v1/information_schema`, so the database picker and SQL autocomplete list every database and table, not just the first page.
+- The query editor's database picker searches by name on the server as you type (`GET /v1/databases?search=`), so any database in a large workspace can be found without loading them all. A saved database shows its name, not its id.
+- Follow pagination on `/v1/information_schema`, so SQL autocomplete lists every table, not just the first page.
 - Handle the Results API's `202` (still persisting) and `409` (failed) responses. Polling keeps its fast 50ms backoff rather than the server's `Retry-After: 1`, which would add a second to large results.
 - The health check validates credentials with one `GET /v1/databases?limit=1` instead of listing every database.
 

pkg/cmd/mockapi/main.go

@@ -15,6 +15,7 @@ import (
 	"log"
 	"math"
 	"net/http"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -141,7 +142,12 @@ func main() {
 	})
 
 	mux.HandleFunc("GET /v1/databases", func(w http.ResponseWriter, r *http.Request) {
-		_, _ = fmt.Fprint(w, `{"databases":[{"id":"dbmock","name":"mock-db","default_catalog":"default","default_schema":"public","created_at":"2026-09-04T00:00:00Z"}],"count":1,"limit":100,"has_more":false}`)
+		// search is a case-insensitive substring filter on the name.
+		if !strings.Contains("mock-db", strings.ToLower(r.URL.Query().Get("search"))) {
+			_, _ = fmt.Fprint(w, `{"databases":[],"count":0,"limit":50,"has_more":false}`)
+			return
+		}
+		_, _ = fmt.Fprint(w, `{"databases":[{"id":"dbmock","name":"mock-db","default_catalog":"default","default_schema":"public","created_at":"2026-09-04T00:00:00Z"}],"count":1,"limit":50,"has_more":false}`)
 	})
 
 	mux.HandleFunc("GET /v1/databases/dbmock", func(w http.ResponseWriter, r *http.Request) {

pkg/hotdata/client.go

@@ -307,21 +307,27 @@ func (c *Client) listPages(ctx context.Context, path string, q url.Values, decod
 	return nil
 }
 
-// ListDatabases returns every database in the workspace, following pages.
-func (c *Client) ListDatabases(ctx context.Context) ([]Database, error) {
-	var all []Database
-	err := c.listPages(ctx, "/v1/databases", url.Values{"limit": {"100"}}, func(r io.Reader) (page, error) {
-		var out struct {
-			page
-			Databases []Database `json:"databases"`
-		}
-		if err := json.NewDecoder(r).Decode(&out); err != nil {
-			return page{}, fmt.Errorf("decoding databases: %w", err)
-		}
-		all = append(all, out.Databases...)
-		return out.page, nil
-	})
-	return all, err
+// SearchDatabases returns one page of up to limit databases, newest first,
+// whose name contains search (case-insensitive; every database when empty).
+// Workspaces can hold thousands of databases, so pickers search on the
+// server rather than listing them all.
+func (c *Client) SearchDatabases(ctx context.Context, search string, limit int) ([]Database, error) {
+	q := url.Values{"limit": {strconv.Itoa(limit)}}
+	if search != "" {
+		q.Set("search", search)
+	}
+	resp, err := c.do(ctx, http.MethodGet, "/v1/databases?"+q.Encode(), "", nil)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = resp.Body.Close() }()
+	var out struct {
+		Databases []Database `json:"databases"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
+		return nil, fmt.Errorf("decoding databases: %w", err)
+	}
+	return out.Databases, nil
 }
 
 // Ping makes the cheapest workspace-scoped call, validating the API key and

pkg/plugin/datasource.go

@@ -279,6 +279,9 @@ func (d *Datasource) CheckHealth(ctx context.Context, req *backend.CheckHealthRe
 	return &backend.CheckHealthResult{Status: backend.HealthStatusOk, Message: msg}, nil
 }
 
+// databasePickerLimit caps how many databases one picker search returns.
+const databasePickerLimit = 50
+
 // CallResource serves the frontend's pickers and SQL autocomplete; the API
 // key never reaches the browser.
 func (d *Datasource) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
@@ -290,10 +293,22 @@ func (d *Datasource) CallResource(ctx context.Context, req *backend.CallResource
 
 	switch req.Path {
 	case "databases":
-		dbs, err := d.client.ListDatabases(ctx)
+		// ?id= resolves one database (the picker's label for a saved id);
+		// otherwise ?search= filters by name on the server.
+		if id := params.Get("id"); id != "" {
+			db, err := d.client.GetDatabase(ctx, id)
+			if err != nil {
+				return sendJSON(sender, http.StatusBadGateway, map[string]string{"error": err.Error()})
+			}
+			return sendJSON(sender, http.StatusOK, []hotdata.Database{db.Database})
+		}
+		dbs, err := d.client.SearchDatabases(ctx, params.Get("search"), databasePickerLimit)
 		if err != nil {
 			return sendJSON(sender, http.StatusBadGateway, map[string]string{"error": err.Error()})
 		}
+		if dbs == nil {
+			dbs = []hotdata.Database{}
+		}
 		return sendJSON(sender, http.StatusOK, dbs)
 
 	case "workspaces":

pkg/plugin/datasource_test.go

@@ -194,13 +194,20 @@ func newContractServer(t *testing.T, opts serverOpts) (*httptest.Server, *atomic
 		w.Header().Set("Content-Type", "application/vnd.apache.arrow.stream")
 		_, _ = w.Write(arrowBody)
 	})
-	// Listings are paginated: the first page points at a second via next_cursor.
+	// search is a case-insensitive name substring filter; limit sizes the page.
 	mux.HandleFunc("GET /v1/databases", func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Query().Get("cursor") == "" {
-			_, _ = w.Write([]byte(`{"databases":[{"id":"dbid123","name":"test","default_catalog":"default","default_schema":"public"}],"count":1,"has_more":true,"next_cursor":"p2"}`))
+		if r.URL.Query().Get("limit") == "" {
+			w.WriteHeader(http.StatusBadRequest)
 			return
 		}
-		_, _ = w.Write([]byte(`{"databases":[{"id":"dbid456","name":"other","default_catalog":"default","default_schema":"main"}],"count":1,"has_more":false}`))
+		switch strings.ToLower(r.URL.Query().Get("search")) {
+		case "":
+			_, _ = w.Write([]byte(`{"databases":[{"id":"dbid123","name":"test","default_catalog":"default","default_schema":"public"},{"id":"dbid456","name":"other","default_catalog":"default","default_schema":"main"}],"count":2,"has_more":true,"next_cursor":"p2"}`))
+		case "oth":
+			_, _ = w.Write([]byte(`{"databases":[{"id":"dbid456","name":"other","default_catalog":"default","default_schema":"main"}],"count":1,"has_more":false}`))
+		default:
+			_, _ = w.Write([]byte(`{"databases":[],"count":0,"has_more":false}`))
+		}
 	})
 	mux.HandleFunc("GET /v1/databases/dbid123", func(w http.ResponseWriter, r *http.Request) {
 		_, _ = w.Write([]byte(`{"id":"dbid123","name":"test","default_connection_id":"conn123","default_catalog":"default","default_schema":"public"}`))
@@ -505,18 +512,41 @@ func TestResources(t *testing.T) {
 		}
 	})
 
-	t.Run("databases", func(t *testing.T) {
+	databases := func(t *testing.T, url string) []hotdata.Database {
+		t.Helper()
 		rec := &resourceRecorder{}
-		err := ds.CallResource(context.Background(), &backend.CallResourceRequest{Path: "databases", URL: "databases"}, rec)
+		err := ds.CallResource(context.Background(), &backend.CallResourceRequest{Path: "databases", URL: url}, rec)
 		if err != nil || rec.status != 200 {
-			t.Fatalf("databases: err=%v status=%d body=%s", err, rec.status, rec.body)
+			t.Fatalf("%s: err=%v status=%d body=%s", url, err, rec.status, rec.body)
 		}
 		var dbs []hotdata.Database
 		if err := json.Unmarshal(rec.body, &dbs); err != nil {
 			t.Fatal(err)
 		}
-		if len(dbs) != 2 || dbs[1].ID != "dbid456" {
-			t.Fatalf("expected both pages of databases, got %+v", dbs)
+		return dbs
+	}
+
+	t.Run("databases", func(t *testing.T) {
+		if dbs := databases(t, "databases"); len(dbs) != 2 {
+			t.Fatalf("expected one page of 2 databases, got %+v", dbs)
+		}
+	})
+
+	t.Run("databases search", func(t *testing.T) {
+		if dbs := databases(t, "databases?search=Oth"); len(dbs) != 1 || dbs[0].ID != "dbid456" {
+			t.Fatalf("expected the matching database, got %+v", dbs)
+		}
+		// No match must encode as [], not null, for the frontend picker.
+		rec := &resourceRecorder{}
+		_ = ds.CallResource(context.Background(), &backend.CallResourceRequest{Path: "databases", URL: "databases?search=zzz"}, rec)
+		if string(rec.body) != "[]" {
+			t.Fatalf("expected [], got %s", rec.body)
+		}
+	})
+
+	t.Run("databases by id", func(t *testing.T) {
+		if dbs := databases(t, "databases?id=dbid123"); len(dbs) != 1 || dbs[0].Name != "test" {
+			t.Fatalf("expected database dbid123, got %+v", dbs)
 		}
 	})
 

src/components/QueryEditor.tsx

@@ -1,4 +1,4 @@
-import React, { useEffect, useRef, useState } from 'react';
+import React, { useCallback, useEffect, useRef, useState } from 'react';
 import { CodeEditor, Combobox, ComboboxOption, InlineField, InlineFieldRow, monacoTypes } from '@grafana/ui';
 import { QueryEditorProps } from '@grafana/data';
 import { DataSource } from '../datasource';
@@ -19,8 +19,16 @@ const DIALECTS: Array<ComboboxOption<Dialect>> = [
   { label: 'Snowflake', value: 'snowflake' },
 ];
 
+const toOption = (db: DatabaseResource): ComboboxOption<string> => ({
+  label: db.name || db.id,
+  description: db.id,
+  value: db.id,
+});
+
 export function QueryEditor({ datasource, query, onChange, onRunQuery }: Props) {
-  const [databases, setDatabases] = useState<Array<ComboboxOption<string>>>([]);
+  // Label for the selected database. Options load per search, so the saved id
+  // is resolved separately to show its name rather than the raw id.
+  const [selected, setSelected] = useState<ComboboxOption<string> | null>(null);
   const databaseIdRef = useRef(query.databaseId);
   const completionsRef = useRef<monacoTypes.IDisposable>();
 
@@ -30,22 +38,38 @@ export function QueryEditor({ datasource, query, onChange, onRunQuery }: Props)
 
   useEffect(() => () => completionsRef.current?.dispose(), []);
 
+  // Workspaces can hold thousands of databases, so the picker searches by
+  // name on the server instead of loading them all.
+  const loadDatabases = useCallback(
+    (search: string) =>
+      datasource
+        .getResource<DatabaseResource[]>('databases', search ? { search } : undefined)
+        .then((dbs) => dbs.map(toOption))
+        // Picker degrades to free-text entry when discovery fails.
+        .catch(() => []),
+    [datasource]
+  );
+
   useEffect(() => {
+    const id = query.databaseId;
+    if (!id) {
+      return;
+    }
     let active = true;
     datasource
-      .getResource<DatabaseResource[]>('databases')
+      .getResource<DatabaseResource[]>('databases', { id })
       .then((dbs) => {
-        if (active) {
-          setDatabases(dbs.map((db) => ({ label: db.name, description: db.id, value: db.id })));
+        if (active && dbs[0]) {
+          setSelected(toOption(dbs[0]));
         }
       })
       .catch(() => {
-        // Picker degrades to free-text entry when discovery fails.
+        // Falls back to showing the raw id.
       });
     return () => {
       active = false;
     };
-  }, [datasource]);
+  }, [datasource, query.databaseId]);
 
   return (
     <>
@@ -61,9 +85,12 @@ export function QueryEditor({ datasource, query, onChange, onRunQuery }: Props)
             isClearable
             createCustomValue
             placeholder="Datasource default"
-            options={databases}
-            value={query.databaseId ?? null}
-            onChange={(v) => onChange({ ...query, databaseId: v?.value })}
+            options={loadDatabases}
+            value={selected?.value === query.databaseId ? selected : (query.databaseId ?? null)}
+            onChange={(v) => {
+              setSelected(v);
+              onChange({ ...query, databaseId: v?.value });
+            }}
           />
         </InlineField>
         <InlineField label="Format" labelWidth={10}>

src/types.ts

@@ -33,7 +33,7 @@ export interface HotdataSecureJsonData {
 /** Shape served by the backend `databases` resource endpoint. */
 export interface DatabaseResource {
   id: string;
-  name: string;
+  name: string | null;
   default_catalog: string;
   default_schema: string;
 }

tests/queryEditor.spec.ts

@@ -25,3 +25,21 @@ test('table query should return the mock series', async ({ panelEditPage, readPr
   await expect(panelEditPage.refreshPanel()).toBeOK();
   await expect(panelEditPage.panel.data).toContainText(['api']);
 });
+
+// The database picker searches by name on the server (mock API: one database,
+// "mock-db"). Combobox options render in a portal listbox, so they are found by
+// role on the page.
+test('database picker should search databases by name', async ({ panelEditPage, readProvisionedDataSource, page }) => {
+  const ds = await readProvisionedDataSource({ fileName: 'datasources.yml' });
+  await panelEditPage.datasource.set(ds.name);
+  const picker = panelEditPage.getQueryEditorRow('A').getByRole('combobox', { name: 'Database' });
+  const option = page.getByRole('option', { name: /mock-db/ });
+  await picker.click();
+  await picker.fill('MOCK');
+  await expect(option).toBeVisible();
+  await picker.fill('no-such-db');
+  await expect(option).toBeHidden();
+  await picker.fill('mock');
+  await option.click();
+  await expect(picker).toHaveValue('mock-db');
+});