diff --git a/Cargo.toml b/Cargo.toml index 4ef90efa..b6cce553 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,6 @@ pre-release-hook = ["git-cliff", "--unreleased", "--tag", "v{{version}}", "--pre publish = false pre-release-replacements = [ { file = "skills/hotdata/SKILL.md", search = "^version: .+", replace = "version: {{version}}", exactly = 1 }, - { file = "skills/hotdata-search/SKILL.md", search = "^version: .+", replace = "version: {{version}}", exactly = 1 }, { file = "skills/hotdata-analytics/SKILL.md", search = "^version: .+", replace = "version: {{version}}", exactly = 1 }, { file = "skills/hotdata-geospatial/SKILL.md", search = "^version: .+", replace = "version: {{version}}", exactly = 1 }, # README.md no longer carries a hardcoded version badge — it tracks GitHub releases. diff --git a/skills/hotdata-analytics/SKILL.md b/skills/hotdata-analytics/SKILL.md index 74ad0b94..f61eba87 100644 --- a/skills/hotdata-analytics/SKILL.md +++ b/skills/hotdata-analytics/SKILL.md @@ -1,6 +1,6 @@ --- name: hotdata-analytics -description: Use this skill when the user wants OLAP-style SQL analytics in Hotdata — aggregations, GROUP BY, JOINs, reporting, exploratory queries, query run history, stored results, or materialized follow-up tables (Chain into managed databases). Activate for "analyze", "aggregate", "rollup", "pivot", "report", "metrics", "GROUP BY", "query history", "past queries", "query runs", "stored results", "materialize", "chain", "intermediate table", or sorted indexes for filters/range scans. Do not load for BM25/vector search or geospatial SQL — use hotdata-search or hotdata-geospatial. Requires the core hotdata skill for tables and auth. +description: Use this skill when the user wants OLAP-style SQL analytics in Hotdata — aggregations, GROUP BY, JOINs, reporting, exploratory queries, query run history, stored results, or materialized follow-up tables (Chain into managed databases). Activate for "analyze", "aggregate", "rollup", "pivot", "report", "metrics", "GROUP BY", "query history", "past queries", "query runs", "stored results", "materialize", "chain", "intermediate table", or sorted indexes for filters/range scans. Do not load for BM25/vector search or geospatial SQL — use the core hotdata skill (search section) or hotdata-geospatial. Requires the core hotdata skill for tables and auth. version: 0.21.0 --- @@ -8,9 +8,9 @@ version: 0.21.0 **OLAP-style analytics** in Hotdata: PostgreSQL-dialect SQL, query execution, run history, stored results, **Chain** materializations, and **sorted** indexes for filters and joins. -**Prerequisites:** Authenticate, workspace, and catalog discovery via the **`hotdata`** skill (`connections`, `tables`, `databases`). +**Prerequisites:** Authenticate, workspace, and catalog discovery via the **`hotdata`** skill (`tables`, `databases`; attach a connection as a queryable catalog with `databases attach`). -**Related skills:** **`hotdata-search`** (BM25, vector, retrieval indexes), **`hotdata-geospatial`** (spatial SQL). +**Related skills:** the core **`hotdata`** skill (BM25/vector search + retrieval indexes), **`hotdata-geospatial`** (spatial SQL). --- @@ -103,7 +103,7 @@ Full procedure: [references/WORKFLOWS.md](references/WORKFLOWS.md). ## Sorted indexes (filters and range scans) -For equality, range, and sort-heavy OLAP — not full-text or vector (see **`hotdata-search`**): +For equality, range, and sort-heavy OLAP — not full-text or vector (see the core **`hotdata`** skill → Search & retrieval indexes): ```bash hotdata indexes create --catalog --schema --table \ diff --git a/skills/hotdata-analytics/references/WORKFLOWS.md b/skills/hotdata-analytics/references/WORKFLOWS.md index eb859d0d..54879bc7 100644 --- a/skills/hotdata-analytics/references/WORKFLOWS.md +++ b/skills/hotdata-analytics/references/WORKFLOWS.md @@ -2,7 +2,7 @@ OLAP-style SQL, **History** (query runs and stored results), and **Chain** (materialized follow-ups). Requires **`hotdata`** for auth, workspaces, and catalog commands. -**Related:** **`hotdata-search`** for BM25/vector indexes and `hotdata search`; **`hotdata`** [WORKFLOWS.md](../../hotdata/references/WORKFLOWS.md) for managed databases. +**Related:** the core **`hotdata`** skill for BM25/vector indexes and `hotdata search`; **`hotdata`** [WORKFLOWS.md](../../hotdata/references/WORKFLOWS.md) for managed databases. --- @@ -24,7 +24,7 @@ hotdata queries - `` — full metadata (timings, `result_id`, snapshot, hashes) and formatted SQL. - If a run has a `result_id`, fetch rows with `hotdata results ` below. -Use history to spot recurring `WHERE`, `JOIN`, `GROUP BY`, or search-style SQL before adding indexes (**`hotdata-search`**) or new Chain tables. +Use history to spot recurring `WHERE`, `JOIN`, `GROUP BY`, or search-style SQL before adding indexes (core **`hotdata`** skill) or new Chain tables. ### Stored results diff --git a/skills/hotdata-geospatial/SKILL.md b/skills/hotdata-geospatial/SKILL.md index c59b3dd5..7e1cdc9d 100644 --- a/skills/hotdata-geospatial/SKILL.md +++ b/skills/hotdata-geospatial/SKILL.md @@ -8,7 +8,7 @@ version: 0.21.0 Hotdata supports a subset of PostGIS-style functions in **PostgreSQL-dialect SQL**. This skill is data-agnostic — apply it to any table with geometry columns. -**Requires the core `hotdata` skill** for auth, workspace, and table discovery. **Related:** **`hotdata-analytics`** (OLAP SQL), **`hotdata-search`** (BM25/vector). +**Requires the core `hotdata` skill** for auth, workspace, table discovery, and **search/retrieval indexes** (BM25/vector). **Related:** **`hotdata-analytics`** (OLAP SQL). ## Running these queries @@ -20,7 +20,7 @@ hotdata query "" [--workspace-id ] [--database ] [--output table|js - **Fully qualify tables** as `..
` (or `..
` for a managed database) — every `
` placeholder below means a qualified name. - **PostgreSQL dialect:** double-quote any non-lowercase identifier (e.g. `"GeoID"`). -- Discover candidate tables/columns with **`hotdata tables list --connection-id `** (connection tables) or **`hotdata databases tables`** (tables inside a managed database) — see core skill. +- Discover candidate tables/columns with **`hotdata tables list`** (all workspace tables; narrow with `--schema`/`--table` wildcards) or **`hotdata databases tables list`** (tables inside a managed database), then inspect column types with **`hotdata tables show `** — see core skill. --- @@ -46,7 +46,7 @@ wkb_geometry_bbox['xmin'] -- ✓ works (wkb_geometry_bbox).xmin -- ✗ not supported ``` -Find these columns by their `Binary` / `Struct` types in `hotdata tables list --connection-id `. +Find these columns by their `Binary` / `Struct` types with `hotdata tables show ` (or scan `hotdata tables list`). --- @@ -155,7 +155,7 @@ SELECT , ST_AsText(ST_Simplify(ST_GeomFromWKB(wkb_geometry), 0.0001)) AS ## Workflow: explore a new geospatial table -1. **Find geometry columns** — `hotdata tables list --connection-id `; look for `Binary` (WKB) / `Struct` (bbox) types. +1. **Find geometry columns** — `hotdata tables list`, then `hotdata tables show `; look for `Binary` (WKB) / `Struct` (bbox) types. 2. **Geometry types** — run the "Geometry types in a table" pattern above. 3. **Coverage / extent** — aggregate the bbox struct: ```sql diff --git a/skills/hotdata-search/SKILL.md b/skills/hotdata-search/SKILL.md deleted file mode 100644 index 53899842..00000000 --- a/skills/hotdata-search/SKILL.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -name: hotdata-search -description: Use this skill when the user wants full-text search, BM25 keyword search, vector similarity search, semantic search, embeddings, or retrieval indexes in Hotdata. Activate for "hotdata search", "BM25", "full-text", "vector search", "semantic search", "similarity", "embedding", "embedding provider", "create an index" (bm25 or vector), "list indexes" for search, or SQL using bm25_search or vector_distance. Do not load for general SQL analytics (aggregations, GROUP BY) or geospatial work — use hotdata-analytics or hotdata-geospatial instead. Requires the core hotdata skill for auth and workspace basics. -version: 0.21.0 ---- - -# Hotdata Search Skill - -Retrieval workloads in Hotdata: **BM25 full-text**, **vector similarity**, and the **indexes** and **embedding providers** that power them. - -**Prerequisites:** Authenticate, set a workspace, and set an active database (`hotdata databases set `) — see the **`hotdata`** skill. Use fully qualified table names: `..
`. - -**Related skills:** **`hotdata-analytics`** (OLAP SQL, query history, materialized chains), **`hotdata-geospatial`** (PostGIS-style functions). - ---- - -## Search CLI - -Both run server-side. `--type` and `--column` are **optional** when the table has exactly one search index — they are inferred automatically. Specify them when multiple indexes exist. - -```bash -# BM25 (requires a BM25 index on the column) -hotdata search "" --table [--type bm25] [--column ] \ - [--select ] [--limit ] [--workspace-id ] [--output table|json|csv] - -# Vector (requires a vector index; server auto-embeds the query text) -hotdata search "" --table [--type vector] [--column ] \ - [--select ] [--limit ] [--workspace-id ] [--output table|json|csv] - -# With active database: schema.table is enough (catalog resolved from active DB) -hotdata search "" --table [--type bm25|vector] [--column ] -``` - -| Type | Behavior | -|------|----------| -| **`bm25`** | Server generates `bm25_search(table, col, 'text')`. Results sort by score (descending). | -| **`vector`** | Pass plain-text query; name the **source text column** (e.g. `title`). Server embeds using the same provider/metric/dimensions as the index. SQL uses `vector_distance(col, 'text')`. Results sort by distance (ascending). | - -- **Inference:** when `--type` or `--column` are omitted, the CLI fetches the table's indexes and selects the only BM25/vector index. If multiple exist, you must specify both flags. -- **Custom embedding model, raw query vector, or no vector index?** Use `hotdata query` directly (e.g. `cosine_distance(col, [])`) — `search` only auto-embeds the query text via the index's own provider. -- **Before search:** create the right index (`indexes create --type bm25` or `--type vector`). See [references/INDEXES.md](references/INDEXES.md). -- Default `--limit` is 10. -- **Active database:** with `hotdata databases set `, you can pass `schema.table` directly (e.g. `--table public.articles`) — the active database's catalog is resolved automatically. Or use the full `catalog.schema.table` form. Do **not** use the internal `__db_` label or raw connection ID prefix — `bm25_search`/`vector_distance` resolve a catalog attached to the active database, so an `__db_…` or `conn…` prefix errors with *catalog … is not attached*. - ---- - -## Indexes (BM25 and vector) - -Create attaches to a table via its `--catalog` alias (a managed-database catalog or a connection name). `list` narrows to the **active database** when one is set; without one it scans the whole workspace. Filter further with `--schema` / `--table`. `delete` **requires all of** `--catalog` + `--schema` + `--table` + `--name`. - -```bash -# List — active-database scope when a DB is set, else whole-workspace scan -hotdata indexes list [--schema ] [--table
] [--workspace-id ] [--output table|json|yaml] - -# Create — by catalog alias (resolves a managed-database catalog or a connection name) -hotdata indexes create --catalog --schema --table
\ - --column --type bm25|vector \ - [--name ] [--metric l2|cosine|dot] [--async] \ - [--embedding-provider-id ] [--dimensions ] [--output-column ] [--description ] - -# Delete — requires --catalog + --schema + --table + --name -hotdata indexes delete --catalog --schema --table
--name -``` - -- **`--type` is required** on create: `bm25` (one or more text columns, comma-separated in `--column`) or `vector` (exactly one column; often embeddings or auto-embedded text). (`sorted` is also a valid `--type`, covered in **`hotdata-analytics`**.) -- **`sorted`** indexes (range/equality for OLAP filters) are documented in **`hotdata-analytics`** — this skill focuses on retrieval types. -- **`--async`:** poll with `hotdata jobs ` (see **`hotdata`** skill **Jobs**). -- **Auto-embedding:** `--type vector` on a **text** column generates embeddings server-side. Optional `--embedding-provider-id`; default output column `{column}_embedding` (override with `--output-column`). - -Full workflow (gather workload → compare existing → create → verify): [references/INDEXES.md](references/INDEXES.md). - ---- - -## Embedding providers - -```bash -hotdata embedding-providers list [--workspace-id ] [--output table|json|yaml] -hotdata embedding-providers get [--workspace-id ] [--output table|json|yaml] -hotdata embedding-providers create --name --provider-type service|local \ - [--config ''] [--provider-api-key | --secret-name ] [--workspace-id ] [--output table|json|yaml] -hotdata embedding-providers update [--name ] [--config ''] [--provider-api-key | --secret-name ] [--workspace-id ] [--output table|json|yaml] -hotdata embedding-providers delete [--workspace-id ] -``` - -- System providers (e.g. `sys_emb_openai`) are pre-configured; use `list` for IDs to pass to `--embedding-provider-id`. -- `--provider-api-key` is the **embedding service** key (not Hotdata `--api-key`). `--secret-name` references an existing secret. - ---- - -## Quick workflow - -1. `hotdata databases set ` — set an active database, then `hotdata tables list` to confirm column types. -2. `hotdata indexes list` — avoid duplicate indexes (scoped to active DB automatically). -3. `hotdata indexes create --catalog --table
--column --type bm25|vector` (add `--async` if large). -4. `hotdata search "..." --table ` — `--type` and `--column` are inferred when there is one search index. -5. Record what exists in **context:DATAMODEL** (core skill) when the workspace should remember index choices. diff --git a/skills/hotdata/SKILL.md b/skills/hotdata/SKILL.md index 767a803f..491396fb 100644 --- a/skills/hotdata/SKILL.md +++ b/skills/hotdata/SKILL.md @@ -1,6 +1,6 @@ --- name: hotdata -description: Use this skill when the user wants to run core hotdata CLI commands — auth, workspaces, managed databases, tables, basic SQL query, database context (context:DATAMODEL), jobs, ingest (pull external data), and skill install. Activate for "run hotdata", "list workspaces", "list databases", "managed database", "load parquet", "list tables", "show table columns", "execute a query", "database context", "context:DATAMODEL", "ingest", "datasource", "import data from", "connect a data source", "connector", "pull data from postgres/mysql/an API/S3 buckets/Iceberg", or general Hotdata CLI usage. For full-text/vector search and retrieval indexes use hotdata-search; for OLAP analytics, query history, stored results, and Chain materializations use hotdata-analytics; for geospatial/GIS use hotdata-geospatial. +description: Use this skill when the user wants to run core hotdata CLI commands — auth, workspaces, managed databases, tables, basic SQL query, database context (context:DATAMODEL), jobs, ingest (pull external data), full-text/vector search and retrieval indexes, and skill install. Activate for "run hotdata", "list workspaces", "list databases", "managed database", "load parquet", "list tables", "show table columns", "execute a query", "database context", "context:DATAMODEL", "ingest", "datasource", "import data from", "connect a data source", "connector", "pull data from postgres/mysql/an API/S3 buckets/Iceberg", "search", "BM25", "full-text", "vector search", "semantic search", "embedding", "create an index", or general Hotdata CLI usage. For OLAP analytics, query history, stored results, and Chain materializations use hotdata-analytics; for geospatial/GIS use hotdata-geospatial. version: 0.21.0 --- @@ -20,8 +20,7 @@ Install all skills with **`hotdata skills install`**. Load specialized skills on | Skill | Use for | |-------|---------| -| **`hotdata`** (this file) | Auth, workspaces, databases, tables, basic `query`, context, jobs, ingest | -| **`hotdata-search`** | BM25, vector search, `hotdata search`, bm25/vector indexes, embedding providers | +| **`hotdata`** (this file) | Auth, workspaces, databases, tables, basic `query`, context, jobs, ingest, **search + retrieval indexes** (BM25/vector, embedding providers) | | **`hotdata-analytics`** | OLAP SQL, aggregations, query/results history, Chain materializations, sorted indexes | | **`hotdata-geospatial`** | PostGIS-style `ST_*`, WKB, spatial joins | @@ -66,13 +65,13 @@ These are **patterns** built from the commands below—not separate CLI subcomma - **Model (`context:DATAMODEL`)** — The shared semantic map of the active database (entities, keys, joins across sources). Store and read it only via database context (`hotdata context list`, then `show DATAMODEL` **only when listed**, `push DATAMODEL`); refresh using `tables list` and `tables show`. For a deep pass (indexes, per-table detail), see [references/MODEL_BUILD.md](references/MODEL_BUILD.md). - **History / Chain / OLAP SQL** — See **`hotdata-analytics`** and [references/WORKFLOWS.md](references/WORKFLOWS.md). -- **Search / retrieval indexes** — See **`hotdata-search`**. +- **Search / retrieval indexes** — See the **Search & retrieval indexes** command section below. Catalog, skill decision tree, epic flows (onboard, chain, retrieval), and managed databases: [references/WORKFLOWS.md](references/WORKFLOWS.md). ## Available Commands -Top-level subcommands (each detailed below): **`auth`**, **`query`**, **`workspaces`**, **`databases`**, **`tables`**, **`skills`**, **`results`**, **`jobs`**, **`ingest`**, **`indexes`**, **`embedding-providers`**, **`search`**, **`queries`**, **`context`**, **`usage`**, **`completions`**, **`upgrade`**. Search, indexes (bm25/vector), and embedding providers are documented in **`hotdata-search`**; query history, results, Chain, and OLAP patterns in **`hotdata-analytics`**. +Top-level subcommands (each detailed below): **`auth`**, **`query`**, **`workspaces`**, **`databases`**, **`tables`**, **`skills`**, **`results`**, **`jobs`**, **`ingest`**, **`indexes`**, **`embedding-providers`**, **`search`**, **`queries`**, **`context`**, **`usage`**, **`completions`**, **`upgrade`**. Search, indexes (bm25/vector), and embedding providers are documented below (**Search & retrieval indexes**); query history, results, Chain, and OLAP patterns in **`hotdata-analytics`**. Global CLI options: **`--api-key`**, **`-v` / `--version`**, **`-h` / `--help`**, **`--no-input`** (disable interactive prompts; commands that require input will error instead — useful in CI or non-TTY environments). Hidden developer flag: **`--debug`** (verbose HTTP logs). @@ -86,7 +85,7 @@ Returns workspaces with `public_id`, `name`, `active`, `favorite`, `provision_st **Managed databases** are Hotdata-owned catalogs you create and populate yourself — no remote source to sync. Query them in SQL as **`..
`**. Prefer **`hotdata databases`** for this workflow. -**Parquet only:** `databases tables load` accepts **parquet** files (local `--file`, remote `--url`, or a pre-staged `--upload-id`). +**Load sources:** `databases tables load` accepts a **parquet** file (local `--file`, remote `--url`, or a pre-staged `--upload-id`) or a **saved query result** (`--result-id`, from `hotdata results` or a query's `[result-id: …]` footer). **Active database:** `hotdata databases set ` saves the active database to config. All `databases tables` subcommands and all `context` commands default to the active database; pass **`--database `** to override per-command. @@ -108,11 +107,11 @@ hotdata databases attach [--database ] [--alias hotdata databases detach [--database ] # Preferred: load by catalog alias (auto-declares table if needed) -hotdata databases load --catalog --table
[--schema public] (--file | --url | --upload-id ) [--workspace-id ] +hotdata databases load --catalog --table
[--schema public] (--file | --url | --upload-id | --result-id ) [--workspace-id ] # Also available via tables subcommand hotdata databases tables list [--database ] [--schema ] [--workspace-id ] [--output table|json|yaml] -hotdata databases tables load
[--database ] [--schema public] (--file | --url | --upload-id ) [--workspace-id ] +hotdata databases tables load
[--database ] [--schema public] (--file | --url | --upload-id | --result-id ) [--workspace-id ] hotdata databases tables delete
[--database ] [--schema public] [--workspace-id ] ``` @@ -123,9 +122,9 @@ hotdata databases tables delete
[--database ] [--schema public] [--w - `unset` — clears the active database from config. - `` — inspect one database (returns id, catalog, name, expires_at). - `delete` — removes the managed database; clears the active-database config if it matched. -- `load` (top-level shorthand) — loads parquet into `--catalog.--schema.--table`. Accepts `--file`, `--url`, or `--upload-id`. If the table was not declared at create time, the CLI automatically deletes and recreates the database with the table declared, then retries the load. +- `load` (top-level shorthand) — loads into `--catalog.--schema.--table`. Accepts `--file`, `--url`, `--upload-id`, or `--result-id` (a saved query result). If the table was not declared at create time, the CLI automatically deletes and recreates the database with the table declared, then retries the load. - `tables list` — lists tables with `TABLE` (`..
`), `SYNCED`, `LAST_SYNC`. Uses active database when `--database` is omitted. -- `tables load` — uploads a local parquet file (`--file`), a remote parquet URL (`--url`), or a pre-staged upload (`--upload-id`) and publishes with **replace** mode. +- `tables load` — loads a local parquet file (`--file`), a remote parquet URL (`--url`), a pre-staged upload (`--upload-id`), or a saved query result (`--result-id`) and publishes with **replace** mode. - `tables delete` — drops a table from the managed database. - `run` — mints a database-scoped JWT (via `POST /v1/auth/database`) and execs `` with `HOTDATA_DATABASE_TOKEN`, `HOTDATA_DATABASE_REFRESH_TOKEN`, `HOTDATA_DATABASE`, `HOTDATA_WORKSPACE`, and `HOTDATA_API_URL` injected. Pass a database id as a group positional (`hotdata databases run ...`) or via `--database `; omit both to auto-create a scratch database using `--name` / `--schema` / `--table` / `--expires-at`. Use this to launch an agent or child process whose API access is scoped to a single database. The minted JWT carries `database`, `workspaces`, `permissions:["read","write"]`, and `source:"database_token"` — read+write within the token's workspace, with that database as the default query scope. The child `hotdata` (or any tool) picks the token up from `HOTDATA_DATABASE_TOKEN`. The session is persisted at `~/.hotdata/database_session.json` (mode `0600`); the child's exit code is propagated. - `attach` — attaches a **connection** as a queryable catalog on a managed database, so the connection's **live** tables become visible inside that database's query scope. Defaults to the active database; target another with `--database`. `--alias` sets the SQL name the catalog answers to (defaults to the connection's name). This is how you query connection tables and **join across sources** — see [Querying across connections](#querying-across-connections-attach). @@ -215,11 +214,11 @@ hotdata query status - **A query runs inside one managed database** (active database or `--database`); with none set it fails *"a database is required."* The scope sees the database's own catalog **plus any attached connection catalogs only**. To query a connection's tables or join across sources, attach the connection first — see [Querying across connections (attach)](#querying-across-connections-attach). - Use `hotdata tables list` and `hotdata tables show` for discovery — not `information_schema` via `query`. (Discovery lists every workspace table; queryability still requires the table's catalog to be in the active database's scope.) - **PostgreSQL dialect.** Quote non-lowercase columns with double quotes. -- Async runs return `query_run_id` → poll with `query status` (do not re-run the same heavy SQL). +- Async runs return `query_run_id` → poll with `query status` (do not re-run the same heavy SQL). Exit codes for scripting: `0` succeeded, `1` failed, `2` still running (poll again), `3` succeeded but the printed result is a truncated preview. - **Large results are complete, not a preview.** The server returns inline rows only up to a bounded cap and persists the full set out-of-band; `hotdata query` transparently fetches the full result, so the printed rows and row count are the complete set. (If the full result can't be retrieved, the CLI prints the preview and a `warning:` to stderr.) - **Backpressure is handled.** Under heavy concurrent load the server may shed a query with HTTP 429 (`OVERLOADED`); the CLI auto-retries (honoring `Retry-After`) before surfacing an error — no manual retry needed. - **OLAP** (aggregations, history, Chain, sorted indexes): **`hotdata-analytics`** skill. -- **Search** (BM25, vector): **`hotdata-search`** skill. +- **Search** (BM25, vector): the **Search & retrieval indexes** command section below. ### Jobs ``` @@ -227,7 +226,7 @@ hotdata jobs list [--workspace-id ] [--job-type ] [--status hotdata jobs [--workspace-id ] [--output table|json|yaml] ``` - `list` shows only active jobs (`pending`, `running`) by default. Use `--all` to see all jobs. -- `--job-type`: `data_refresh_table`, `data_refresh_connection`, `create_index`. +- `--job-type`: `data_refresh_table`, `data_refresh_connection`, `create_index`, `managed_load`. - `--status`: `pending`, `running`, `succeeded`, `partially_succeeded`, `failed`. - Use `hotdata jobs ` to inspect a specific job's status, error, and result. @@ -292,6 +291,61 @@ Agent tips: - Table names per type: `buckets` datasources expose one table named `files` (`FROM ` or `FROM .files`); iceberg tables are addressed by the exact onboarded name (`FROM ice."ns.orders"`), the underscored form (`FROM ice.ns_orders`), or the bare table name — `show-datasource` lists what was discovered. - The pre-0.14 `*-connection` verbs still work as hidden aliases of the `*-datasource` ones. +### Search & retrieval indexes (`search`, `indexes`, `embedding-providers`) + +BM25 full-text and vector similarity search, plus the indexes and embedding providers that power them. Set an active database (`hotdata databases set `) and use fully-qualified `..
` names (or `schema.table` with an active DB). Sorted indexes for OLAP range/equality filters are in **`hotdata-analytics`**. + +**`search`** — both run server-side. `--type` and `--column` are optional when the table has exactly one search index (inferred); specify them when multiple exist. + +```bash +# BM25 (requires a BM25 index on the column) +hotdata search "" --table [--type bm25] [--column ] \ + [--select ] [--limit ] [--workspace-id ] [--output table|json|csv] + +# Vector (requires a vector index; server auto-embeds the query text) +hotdata search "" --table [--type vector] [--column ] \ + [--select ] [--limit ] [--workspace-id ] [--output table|json|csv] +``` + +| Type | Behavior | +|------|----------| +| **`bm25`** | Server generates `bm25_search(table, col, 'text')`. Results sort by score (descending). | +| **`vector`** | Pass a plain-text query; name the **source text column** (e.g. `title`). Server embeds using the index's provider/metric/dimensions. SQL uses `vector_distance(col, 'text')`. Results sort by distance (ascending). | + +- **Inference:** with `--type`/`--column` omitted, the CLI selects the table's only BM25/vector index; if multiple exist, pass both. Default `--limit` is 10. +- **Custom embedding model / raw query vector / no vector index?** Use `hotdata query` directly (e.g. `cosine_distance(col, [])`) — `search` only auto-embeds via the index's own provider. +- Do **not** use the internal `__db_` / `conn…` label — `bm25_search`/`vector_distance` resolve a catalog attached to the active database, so those prefixes error with *catalog … is not attached*. + +**`indexes`** (BM25 and vector) — `create` attaches to a table via its `--catalog` alias (a managed-database catalog or connection name). `list` scopes to the active database when set, else the whole workspace (narrow with `--schema`/`--table`). `delete` **requires** `--catalog` + `--schema` + `--table` + `--name`. + +```bash +hotdata indexes list [--schema ] [--table
] [--workspace-id ] [--output table|json|yaml] +hotdata indexes create --catalog --schema --table
\ + --column --type bm25|vector \ + [--name ] [--metric l2|cosine|dot] [--async] \ + [--embedding-provider-id ] [--dimensions ] [--output-column ] [--description ] +hotdata indexes delete --catalog --schema --table
--name +``` + +- **`--type` required** on create: `bm25` (one or more text columns, comma-separated in `--column`) or `vector` (exactly one column). `sorted` is also valid (OLAP filters — see **`hotdata-analytics`**). +- **`--async`:** poll with `hotdata jobs ` (see **Jobs** above). +- **Auto-embedding:** `--type vector` on a **text** column embeds server-side; default output column `{column}_embedding` (override with `--output-column`). +- Full workflow (gather workload → compare → create → verify): [references/INDEXES.md](references/INDEXES.md). + +**`embedding-providers`** — the providers vector indexes embed with. + +```bash +hotdata embedding-providers list [--workspace-id ] [--output table|json|yaml] +hotdata embedding-providers get [--workspace-id ] [--output table|json|yaml] +hotdata embedding-providers create --name --provider-type service|local \ + [--config ''] [--provider-api-key | --secret-name ] [--workspace-id ] [--output table|json|yaml] +hotdata embedding-providers update [--name ] [--config ''] [--provider-api-key | --secret-name ] [--workspace-id ] +hotdata embedding-providers delete [--workspace-id ] +``` + +- System providers (e.g. `sys_emb_openai`) are pre-configured; `list` for IDs to pass to `--embedding-provider-id`. +- `--provider-api-key` is the **embedding service** key (not Hotdata `--api-key`); `--secret-name` references an existing secret. + ### Usage ``` hotdata usage [--since ] [--workspace-id ] [--output table|json|yaml] @@ -303,7 +357,7 @@ Workspace usage for the current billing window (or since `--since`): `query_coun ### Agent skills (`skills`) -Bundled Markdown skills (**`hotdata`**, **`hotdata-search`**, **`hotdata-analytics`**, **`hotdata-geospatial`**) ship with the CLI release tarball. +Bundled Markdown skills (**`hotdata`**, **`hotdata-analytics`**, **`hotdata-geospatial`**) ship with the CLI release tarball. ``` hotdata skills install [--project] diff --git a/skills/hotdata/references/DATA_MODEL.template.md b/skills/hotdata/references/DATA_MODEL.template.md index 99c25475..73cde717 100644 --- a/skills/hotdata/references/DATA_MODEL.template.md +++ b/skills/hotdata/references/DATA_MODEL.template.md @@ -60,7 +60,7 @@ Document safe join paths and caveats (fan-out, timing, different refresh cadence |-------|--------|--------------------------|--------------|-------| | | | | | | -_Use `hotdata indexes list` (no flags — covers connection tables **and** managed databases; see **hotdata-search** skill). Record bm25/vector indexes here; sorted indexes for OLAP filters in **hotdata-analytics**._ +_Use `hotdata indexes list` (no flags — covers connection tables **and** managed databases; see the core **hotdata** skill → Search & retrieval indexes). Record bm25/vector indexes here; sorted indexes for OLAP filters in **hotdata-analytics**._ ## Managed databases (uploaded) diff --git a/skills/hotdata-search/references/INDEXES.md b/skills/hotdata/references/INDEXES.md similarity index 100% rename from skills/hotdata-search/references/INDEXES.md rename to skills/hotdata/references/INDEXES.md diff --git a/skills/hotdata/references/MODEL_BUILD.md b/skills/hotdata/references/MODEL_BUILD.md index a007d722..ab48c18f 100644 --- a/skills/hotdata/references/MODEL_BUILD.md +++ b/skills/hotdata/references/MODEL_BUILD.md @@ -100,7 +100,7 @@ Note: - **Time** columns — event grain vs slowly changing dimensions. - **Facts vs dimensions** — for analytics-oriented workspaces. -When suggesting a new index, use the same connection/schema/table/column names as in `tables list` and **`hotdata-search`** / **`hotdata-analytics`** `indexes create` examples (bm25/vector vs sorted). +When suggesting a new index, use the same connection/schema/table/column names as in `tables list` and the core **`hotdata`** / **`hotdata-analytics`** `indexes create` examples (bm25/vector vs sorted). --- diff --git a/skills/hotdata/references/WORKFLOWS.md b/skills/hotdata/references/WORKFLOWS.md index cf367f65..23f7e10d 100644 --- a/skills/hotdata/references/WORKFLOWS.md +++ b/skills/hotdata/references/WORKFLOWS.md @@ -13,7 +13,7 @@ Load **`hotdata`** first for auth and workspace setup. Add a sub-skill only when | Login, workspaces, connections, tables, context | **`hotdata`** | `auth`, `workspaces`, `connections`, `tables`, `context` | | Load parquet files into a managed database | **`hotdata`** | `databases create` + `databases load` | | SQL analytics, aggregations, history, Chain | **`hotdata-analytics`** | `query`, `queries`, `results` | -| BM25 / vector search, retrieval indexes | **`hotdata-search`** | `search`, `indexes create`, `embedding-providers` | +| BM25 / vector search, retrieval indexes | core **`hotdata`** (Search & retrieval indexes) | `search`, `indexes create`, `embedding-providers` | | Geospatial / PostGIS-style SQL | **`hotdata-geospatial`** | `query` with `ST_*`, WKB columns | | Concept | Where documented | @@ -21,7 +21,7 @@ Load **`hotdata`** first for auth and workspace setup. Add a sub-skill only when | **Model** | This file — [Model](#model) | | **Upload path (managed databases)** | This file — [Managed databases](#managed-databases) | | **History / Chain** | **`hotdata-analytics`** — [WORKFLOWS.md](../../hotdata-analytics/references/WORKFLOWS.md) | -| **Search indexes** | **`hotdata-search`** — [INDEXES.md](../../hotdata-search/references/INDEXES.md) | +| **Search indexes** | core **`hotdata`** skill (Search & retrieval indexes) — [INDEXES.md](INDEXES.md) | | **Epic flows** | This file — [Epic flows](#epic-flows) | --- @@ -59,7 +59,7 @@ End-to-end checklists. Use the linked sections for command detail and guardrails ### Retrieval (index then search) -**Skill:** **`hotdata-search`** (schema via **`hotdata`**) +**Skill:** core **`hotdata`** (Search & retrieval indexes) 1. [ ] `hotdata tables list --connection-id ` — pick text column (BM25) or embedding/text column (vector) 2. [ ] `hotdata indexes list` — avoid duplicate bm25/vector indexes on the same column @@ -72,7 +72,7 @@ End-to-end checklists. Use the linked sections for command detail and guardrails - [ ] `hotdata search "…" --table … --type bm25 --column ` (explicit) 5. [ ] (Optional) Note indexes in **context:DATAMODEL → Search & index summary** -**Detail:** [hotdata-search INDEXES.md](../../hotdata-search/references/INDEXES.md) +**Detail:** [INDEXES.md](INDEXES.md) ### Cross-source query (attach a connection) diff --git a/src/commands/skill.rs b/src/commands/skill.rs index 8785d894..d2307992 100644 --- a/src/commands/skill.rs +++ b/src/commands/skill.rs @@ -21,12 +21,7 @@ pub enum SkillCommands { const REPO: &str = "hotdata-dev/hotdata-cli"; const PRIMARY_SKILL_NAME: &str = "hotdata"; -const SKILL_NAMES: &[&str] = &[ - "hotdata", - "hotdata-search", - "hotdata-analytics", - "hotdata-geospatial", -]; +const SKILL_NAMES: &[&str] = &["hotdata", "hotdata-analytics", "hotdata-geospatial"]; const CURRENT_VERSION: &str = env!("CARGO_PKG_VERSION"); /// Agent root directories to check for symlink installation.