From 7f4c13790ab2c59b31560c4c7072531dff394a1a Mon Sep 17 00:00:00 2001 From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:55:10 -0700 Subject: [PATCH] Search databases on the server in the query editor picker Listing every database took ~935ms on a 1,594-database workspace and grows with workspace size. The picker now loads options per keystroke from GET /v1/databases?search= (one page of 50, ~70ms), and resolves a saved database id to its name via ?id=. --- CHANGELOG.md | 3 ++- pkg/cmd/mockapi/main.go | 8 +++++- pkg/hotdata/client.go | 36 ++++++++++++++----------- pkg/plugin/datasource.go | 17 +++++++++++- pkg/plugin/datasource_test.go | 48 +++++++++++++++++++++++++++------- src/components/QueryEditor.tsx | 47 ++++++++++++++++++++++++++------- src/types.ts | 2 +- tests/queryEditor.spec.ts | 18 +++++++++++++ 8 files changed, 141 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60666dd..4fbdaf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ - Read query results as Arrow directly from `POST /v1/query`, so a fast query takes one request instead of two. Async and too-large results still come from the Results API. - Show the real error for failed queries: read `error_message` from query runs and failed results, and show the `message` and `code` from Hotdata error bodies instead of raw JSON. -- Follow pagination on `/v1/databases` and `/v1/information_schema`, so the database picker and SQL autocomplete list every database and table, not just the first page. +- The query editor's database picker searches by name on the server as you type (`GET /v1/databases?search=`), so any database in a large workspace can be found without loading them all. A saved database shows its name, not its id. +- Follow pagination on `/v1/information_schema`, so SQL autocomplete lists every table, not just the first page. - Handle the Results API's `202` (still persisting) and `409` (failed) responses. Polling keeps its fast 50ms backoff rather than the server's `Retry-After: 1`, which would add a second to large results. - The health check validates credentials with one `GET /v1/databases?limit=1` instead of listing every database. diff --git a/pkg/cmd/mockapi/main.go b/pkg/cmd/mockapi/main.go index de1c1da..76eeafe 100644 --- a/pkg/cmd/mockapi/main.go +++ b/pkg/cmd/mockapi/main.go @@ -15,6 +15,7 @@ import ( "log" "math" "net/http" + "strings" "sync" "sync/atomic" "time" @@ -141,7 +142,12 @@ func main() { }) mux.HandleFunc("GET /v1/databases", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, `{"databases":[{"id":"dbmock","name":"mock-db","default_catalog":"default","default_schema":"public","created_at":"2026-09-04T00:00:00Z"}],"count":1,"limit":100,"has_more":false}`) + // search is a case-insensitive substring filter on the name. + if !strings.Contains("mock-db", strings.ToLower(r.URL.Query().Get("search"))) { + _, _ = fmt.Fprint(w, `{"databases":[],"count":0,"limit":50,"has_more":false}`) + return + } + _, _ = fmt.Fprint(w, `{"databases":[{"id":"dbmock","name":"mock-db","default_catalog":"default","default_schema":"public","created_at":"2026-09-04T00:00:00Z"}],"count":1,"limit":50,"has_more":false}`) }) mux.HandleFunc("GET /v1/databases/dbmock", func(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/hotdata/client.go b/pkg/hotdata/client.go index 4640af8..7d7006a 100644 --- a/pkg/hotdata/client.go +++ b/pkg/hotdata/client.go @@ -307,21 +307,27 @@ func (c *Client) listPages(ctx context.Context, path string, q url.Values, decod return nil } -// ListDatabases returns every database in the workspace, following pages. -func (c *Client) ListDatabases(ctx context.Context) ([]Database, error) { - var all []Database - err := c.listPages(ctx, "/v1/databases", url.Values{"limit": {"100"}}, func(r io.Reader) (page, error) { - var out struct { - page - Databases []Database `json:"databases"` - } - if err := json.NewDecoder(r).Decode(&out); err != nil { - return page{}, fmt.Errorf("decoding databases: %w", err) - } - all = append(all, out.Databases...) - return out.page, nil - }) - return all, err +// SearchDatabases returns one page of up to limit databases, newest first, +// whose name contains search (case-insensitive; every database when empty). +// Workspaces can hold thousands of databases, so pickers search on the +// server rather than listing them all. +func (c *Client) SearchDatabases(ctx context.Context, search string, limit int) ([]Database, error) { + q := url.Values{"limit": {strconv.Itoa(limit)}} + if search != "" { + q.Set("search", search) + } + resp, err := c.do(ctx, http.MethodGet, "/v1/databases?"+q.Encode(), "", nil) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + var out struct { + Databases []Database `json:"databases"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, fmt.Errorf("decoding databases: %w", err) + } + return out.Databases, nil } // Ping makes the cheapest workspace-scoped call, validating the API key and diff --git a/pkg/plugin/datasource.go b/pkg/plugin/datasource.go index 5e5e332..7c43c24 100644 --- a/pkg/plugin/datasource.go +++ b/pkg/plugin/datasource.go @@ -279,6 +279,9 @@ func (d *Datasource) CheckHealth(ctx context.Context, req *backend.CheckHealthRe return &backend.CheckHealthResult{Status: backend.HealthStatusOk, Message: msg}, nil } +// databasePickerLimit caps how many databases one picker search returns. +const databasePickerLimit = 50 + // CallResource serves the frontend's pickers and SQL autocomplete; the API // key never reaches the browser. func (d *Datasource) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { @@ -290,10 +293,22 @@ func (d *Datasource) CallResource(ctx context.Context, req *backend.CallResource switch req.Path { case "databases": - dbs, err := d.client.ListDatabases(ctx) + // ?id= resolves one database (the picker's label for a saved id); + // otherwise ?search= filters by name on the server. + if id := params.Get("id"); id != "" { + db, err := d.client.GetDatabase(ctx, id) + if err != nil { + return sendJSON(sender, http.StatusBadGateway, map[string]string{"error": err.Error()}) + } + return sendJSON(sender, http.StatusOK, []hotdata.Database{db.Database}) + } + dbs, err := d.client.SearchDatabases(ctx, params.Get("search"), databasePickerLimit) if err != nil { return sendJSON(sender, http.StatusBadGateway, map[string]string{"error": err.Error()}) } + if dbs == nil { + dbs = []hotdata.Database{} + } return sendJSON(sender, http.StatusOK, dbs) case "workspaces": diff --git a/pkg/plugin/datasource_test.go b/pkg/plugin/datasource_test.go index dfb5bac..10eb243 100644 --- a/pkg/plugin/datasource_test.go +++ b/pkg/plugin/datasource_test.go @@ -194,13 +194,20 @@ func newContractServer(t *testing.T, opts serverOpts) (*httptest.Server, *atomic w.Header().Set("Content-Type", "application/vnd.apache.arrow.stream") _, _ = w.Write(arrowBody) }) - // Listings are paginated: the first page points at a second via next_cursor. + // search is a case-insensitive name substring filter; limit sizes the page. mux.HandleFunc("GET /v1/databases", func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("cursor") == "" { - _, _ = w.Write([]byte(`{"databases":[{"id":"dbid123","name":"test","default_catalog":"default","default_schema":"public"}],"count":1,"has_more":true,"next_cursor":"p2"}`)) + if r.URL.Query().Get("limit") == "" { + w.WriteHeader(http.StatusBadRequest) return } - _, _ = w.Write([]byte(`{"databases":[{"id":"dbid456","name":"other","default_catalog":"default","default_schema":"main"}],"count":1,"has_more":false}`)) + switch strings.ToLower(r.URL.Query().Get("search")) { + case "": + _, _ = w.Write([]byte(`{"databases":[{"id":"dbid123","name":"test","default_catalog":"default","default_schema":"public"},{"id":"dbid456","name":"other","default_catalog":"default","default_schema":"main"}],"count":2,"has_more":true,"next_cursor":"p2"}`)) + case "oth": + _, _ = w.Write([]byte(`{"databases":[{"id":"dbid456","name":"other","default_catalog":"default","default_schema":"main"}],"count":1,"has_more":false}`)) + default: + _, _ = w.Write([]byte(`{"databases":[],"count":0,"has_more":false}`)) + } }) mux.HandleFunc("GET /v1/databases/dbid123", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(`{"id":"dbid123","name":"test","default_connection_id":"conn123","default_catalog":"default","default_schema":"public"}`)) @@ -505,18 +512,41 @@ func TestResources(t *testing.T) { } }) - t.Run("databases", func(t *testing.T) { + databases := func(t *testing.T, url string) []hotdata.Database { + t.Helper() rec := &resourceRecorder{} - err := ds.CallResource(context.Background(), &backend.CallResourceRequest{Path: "databases", URL: "databases"}, rec) + err := ds.CallResource(context.Background(), &backend.CallResourceRequest{Path: "databases", URL: url}, rec) if err != nil || rec.status != 200 { - t.Fatalf("databases: err=%v status=%d body=%s", err, rec.status, rec.body) + t.Fatalf("%s: err=%v status=%d body=%s", url, err, rec.status, rec.body) } var dbs []hotdata.Database if err := json.Unmarshal(rec.body, &dbs); err != nil { t.Fatal(err) } - if len(dbs) != 2 || dbs[1].ID != "dbid456" { - t.Fatalf("expected both pages of databases, got %+v", dbs) + return dbs + } + + t.Run("databases", func(t *testing.T) { + if dbs := databases(t, "databases"); len(dbs) != 2 { + t.Fatalf("expected one page of 2 databases, got %+v", dbs) + } + }) + + t.Run("databases search", func(t *testing.T) { + if dbs := databases(t, "databases?search=Oth"); len(dbs) != 1 || dbs[0].ID != "dbid456" { + t.Fatalf("expected the matching database, got %+v", dbs) + } + // No match must encode as [], not null, for the frontend picker. + rec := &resourceRecorder{} + _ = ds.CallResource(context.Background(), &backend.CallResourceRequest{Path: "databases", URL: "databases?search=zzz"}, rec) + if string(rec.body) != "[]" { + t.Fatalf("expected [], got %s", rec.body) + } + }) + + t.Run("databases by id", func(t *testing.T) { + if dbs := databases(t, "databases?id=dbid123"); len(dbs) != 1 || dbs[0].Name != "test" { + t.Fatalf("expected database dbid123, got %+v", dbs) } }) diff --git a/src/components/QueryEditor.tsx b/src/components/QueryEditor.tsx index 127807c..da8455e 100644 --- a/src/components/QueryEditor.tsx +++ b/src/components/QueryEditor.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useRef, useState } from 'react'; +import React, { useCallback, useEffect, useRef, useState } from 'react'; import { CodeEditor, Combobox, ComboboxOption, InlineField, InlineFieldRow, monacoTypes } from '@grafana/ui'; import { QueryEditorProps } from '@grafana/data'; import { DataSource } from '../datasource'; @@ -19,8 +19,16 @@ const DIALECTS: Array> = [ { label: 'Snowflake', value: 'snowflake' }, ]; +const toOption = (db: DatabaseResource): ComboboxOption => ({ + label: db.name || db.id, + description: db.id, + value: db.id, +}); + export function QueryEditor({ datasource, query, onChange, onRunQuery }: Props) { - const [databases, setDatabases] = useState>>([]); + // Label for the selected database. Options load per search, so the saved id + // is resolved separately to show its name rather than the raw id. + const [selected, setSelected] = useState | null>(null); const databaseIdRef = useRef(query.databaseId); const completionsRef = useRef(); @@ -30,22 +38,38 @@ export function QueryEditor({ datasource, query, onChange, onRunQuery }: Props) useEffect(() => () => completionsRef.current?.dispose(), []); + // Workspaces can hold thousands of databases, so the picker searches by + // name on the server instead of loading them all. + const loadDatabases = useCallback( + (search: string) => + datasource + .getResource('databases', search ? { search } : undefined) + .then((dbs) => dbs.map(toOption)) + // Picker degrades to free-text entry when discovery fails. + .catch(() => []), + [datasource] + ); + useEffect(() => { + const id = query.databaseId; + if (!id) { + return; + } let active = true; datasource - .getResource('databases') + .getResource('databases', { id }) .then((dbs) => { - if (active) { - setDatabases(dbs.map((db) => ({ label: db.name, description: db.id, value: db.id }))); + if (active && dbs[0]) { + setSelected(toOption(dbs[0])); } }) .catch(() => { - // Picker degrades to free-text entry when discovery fails. + // Falls back to showing the raw id. }); return () => { active = false; }; - }, [datasource]); + }, [datasource, query.databaseId]); return ( <> @@ -61,9 +85,12 @@ export function QueryEditor({ datasource, query, onChange, onRunQuery }: Props) isClearable createCustomValue placeholder="Datasource default" - options={databases} - value={query.databaseId ?? null} - onChange={(v) => onChange({ ...query, databaseId: v?.value })} + options={loadDatabases} + value={selected?.value === query.databaseId ? selected : (query.databaseId ?? null)} + onChange={(v) => { + setSelected(v); + onChange({ ...query, databaseId: v?.value }); + }} /> diff --git a/src/types.ts b/src/types.ts index 2ca93ba..d1d9470 100644 --- a/src/types.ts +++ b/src/types.ts @@ -33,7 +33,7 @@ export interface HotdataSecureJsonData { /** Shape served by the backend `databases` resource endpoint. */ export interface DatabaseResource { id: string; - name: string; + name: string | null; default_catalog: string; default_schema: string; } diff --git a/tests/queryEditor.spec.ts b/tests/queryEditor.spec.ts index d7adeaf..97f7cfa 100644 --- a/tests/queryEditor.spec.ts +++ b/tests/queryEditor.spec.ts @@ -25,3 +25,21 @@ test('table query should return the mock series', async ({ panelEditPage, readPr await expect(panelEditPage.refreshPanel()).toBeOK(); await expect(panelEditPage.panel.data).toContainText(['api']); }); + +// The database picker searches by name on the server (mock API: one database, +// "mock-db"). Combobox options render in a portal listbox, so they are found by +// role on the page. +test('database picker should search databases by name', async ({ panelEditPage, readProvisionedDataSource, page }) => { + const ds = await readProvisionedDataSource({ fileName: 'datasources.yml' }); + await panelEditPage.datasource.set(ds.name); + const picker = panelEditPage.getQueryEditorRow('A').getByRole('combobox', { name: 'Database' }); + const option = page.getByRole('option', { name: /mock-db/ }); + await picker.click(); + await picker.fill('MOCK'); + await expect(option).toBeVisible(); + await picker.fill('no-such-db'); + await expect(option).toBeHidden(); + await picker.fill('mock'); + await option.click(); + await expect(picker).toHaveValue('mock-db'); +});