feat: retry locked loads, read env vars, clear workspace error

.github/workflows/regenerate.yml

@@ -201,6 +201,12 @@ jobs:
       - name: Patch transport request defaults (Accept-Encoding, User-Agent)
         run: python3 scripts/patch_request_defaults.py
 
+      - name: Patch missing-workspace guard into ApiClient
+        run: python3 scripts/patch_workspace_guard.py
+
+      - name: Patch request models to reject unknown fields (extra=forbid)
+        run: python3 scripts/patch_model_extra_forbid.py
+
       # The API-token -> JWT key exchange is deprecated: the configured API token
       # is the bearer credential and goes on the wire verbatim. This check is the
       # inverse of the old "did the exchange survive?" guard — it fails if the

.openapi-generator-ignore

@@ -11,11 +11,16 @@ setup.py
 # connection-reset retry on all methods, #118), uploads.py (transparent
 # presigned direct-to-storage upload flow), _useragent.py (runtime-resolved
 # User-Agent; the generator's httpUserAgent property can only bake a literal at
-# generation time, which goes stale between releases).
+# generation time, which goes stale between releases), loads.py (409
+# RESOURCE_LOCKED retry of table loads while another load holds the table),
+# _workspace.py (client-side missing-workspace guard, wired into api_client by
+# scripts/patch_workspace_guard.py).
 hotdata/arrow.py
 hotdata/query.py
 hotdata/_retry.py
 hotdata/uploads.py
+hotdata/loads.py
+hotdata/_workspace.py
 hotdata/_useragent.py
 
 # Hand-written test for the patched ApiClient.close()/context-manager behavior

.openapi-generator-templates/configuration.mustache

@@ -9,6 +9,7 @@ import logging
 from logging import FileHandler
 {{^async}}
 import multiprocessing
+import os
 {{/async}}
 import sys
 from importlib.metadata import PackageNotFoundError, version as _pkg_version
@@ -156,13 +157,16 @@ class HostSetting(TypedDict):
 class Configuration:
     """This class contains various settings of the API client.
 
-    :param host: Base url.
+    :param host: Base url. Defaults to the `HOTDATA_API_URL` environment
+      variable when set, else `https://api.hotdata.dev`.
     :param ignore_operation_servers
       Boolean to ignore operation servers for the API client.
       Config will use `host` as the base url regardless of the operation servers.
     :param api_key: Hotdata API key, sent as `Authorization: Bearer <key>`.
+      Defaults to the `HOTDATA_API_KEY` environment variable when not given.
     :param workspace_id: Public id of the target workspace, sent as
-      `X-Workspace-Id`.
+      `X-Workspace-Id`. Defaults to the `HOTDATA_WORKSPACE` environment
+      variable when not given.
     :param api_keys: Escape hatch for raw apiKey-scheme values, keyed by
       security scheme name. Prefer `workspace_id`.
     :param api_key_prefix: Dict to store API prefix (e.g. Bearer).
@@ -299,6 +303,14 @@ conf = {{packageName}}.Configuration(
     ) -> None:
         """Constructor
         """
+        # Environment defaults, matching the Hotdata CLI: an explicit argument
+        # always wins; an unset or empty variable leaves the default in place.
+        if api_key is None:
+            api_key = os.environ.get("HOTDATA_API_KEY") or None
+        if workspace_id is None:
+            workspace_id = os.environ.get("HOTDATA_WORKSPACE") or None
+        if host is None:
+            host = os.environ.get("HOTDATA_API_URL") or None
         self._base_path = "{{basePath}}" if host is None else host
         """Default Base url
         """

CHANGELOG.md

@@ -7,8 +7,44 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- `hotdata.Configuration` reads `HOTDATA_API_KEY`, `HOTDATA_WORKSPACE` and
+  `HOTDATA_API_URL` from the environment when `api_key`, `workspace_id` or
+  `host` is not passed, matching the CLI. Explicit arguments always win and an
+  empty variable counts as unset.
+- A workspace-scoped call made with no workspace id configured now raises
+  `hotdata.MissingWorkspaceError` (an `ApiValueError`) before anything is sent,
+  naming `workspace_id=` and `HOTDATA_WORKSPACE`. Previously the request went
+  out without `X-Workspace-Id` and the server answered a bare 404. Calls that
+  pass the header via `_headers`, use `_request_auth`, or target endpoints that
+  are not workspace-scoped (the Workspaces endpoints) are unaffected.
+- `QueryApi.fetch_ready_result(result_id, x_database_id, offset=, limit=,
+  format=)` waits for a result to be `ready` and then fetches its rows in one
+  call. `QueryApi.wait_for_result` is unchanged: it polls with `limit=0` and
+  returns a status-only response with no rows, which its docstring now states
+  up front.
+- `DatabasesApi.load_database_table` and `ConnectionsApi.load_managed_table`
+  (the classes exported from `hotdata`) retry automatically while another load
+  holds the table: HTTP 409 with error code `RESOURCE_LOCKED` is retried,
+  honouring `Retry-After` (else exponential backoff) with jitter, under a
+  60s budget by default. Tune or disable with `LoadRetryPolicy`
+  (`DatabasesApi(client, load_retry=LoadRetryPolicy(...))`); when the budget
+  runs out, `hotdata.TableLockedError` (a 409 `ApiException`) is raised. Other
+  409s (`CONFLICT`, `IDEMPOTENCY_KEY_CONFLICT`) are never retried. The raw
+  generated classes stay available under `hotdata.api.*`.
+
 ### Changed
 
+- **Breaking:** request models reject unknown fields. Every model used as a
+  request body (`QueryRequest`, `LoadManagedTableRequest`,
+  `CreateDatabaseRequest`, ... and the models nested in them) now carries
+  `extra="forbid"`, so a misspelled keyword, or a field a newer API has that
+  this SDK release does not know yet, raises `pydantic.ValidationError` at
+  construction instead of being silently dropped from the request. Upgrade the
+  SDK to use a new field. Response models are unchanged: unknown fields in
+  server responses are still ignored, so a newer server never breaks an older
+  SDK.
 - feat(query): add arrow ipc stream response format
 - chore(query): clarify write statement handling in dialect parameter
 - chore(indexes): clarify vector index constraints and algorithms

README.md

@@ -37,7 +37,17 @@ configuration = hotdata.Configuration(
 )
 ```
 
-`host` defaults to `https://api.hotdata.dev`. Override it if you target another environment.
+Each of these can come from the environment instead, the same variables the
+CLI uses: `HOTDATA_API_KEY`, `HOTDATA_WORKSPACE` and `HOTDATA_API_URL` fill in
+`api_key`, `workspace_id` and `host` when the argument is not passed. An
+explicit argument always wins.
+
+`host` defaults to `https://api.hotdata.dev`. Override it (or set
+`HOTDATA_API_URL`) if you target another environment.
+
+Almost every call is workspace-scoped. If no workspace id is configured, the
+SDK raises `hotdata.MissingWorkspaceError` before sending the request rather
+than letting the server answer with an unexplained 404.
 
 ## Usage
 
@@ -60,6 +70,25 @@ with hotdata.ApiClient(configuration) as api_client:
 
 Each `Api` class groups endpoints by resource. Construct the client, then call the typed methods you need.
 
+## Loading tables
+
+Loads are serialised per table on the server. `hotdata.DatabasesApi` and
+`hotdata.ConnectionsApi` retry a load that hits a table another load still
+holds (HTTP 409, error code `RESOURCE_LOCKED`), honouring the server's
+`Retry-After`, for up to 60s by default:
+
+```python
+from hotdata import DatabasesApi, LoadRetryPolicy, TableLockedError
+
+databases = DatabasesApi(api_client, load_retry=LoadRetryPolicy(deadline_s=120))
+try:
+    databases.load_database_table(db_id, "public", "events", request)
+except TableLockedError as e:
+    print(f"gave up after {e.attempts} attempts: {e.reason}")
+```
+
+Other 409s (`CONFLICT`, `IDEMPOTENCY_KEY_CONFLICT`) are raised immediately.
+
 ## Arrow results
 
 Query results can be fetched as an [Apache Arrow](https://arrow.apache.org/) IPC stream instead of JSON, which is faster and far more memory-efficient for large result sets. Install the optional extra:

hotdata/__init__.py

@@ -312,10 +312,21 @@
 # hotdata.query.QueryApi adds 429 retry + truncation auto-follow;
 # hotdata.arrow.ResultsApi adds Arrow IPC result fetch;
 # hotdata.uploads.UploadsApi adds transparent presigned direct-to-storage
-# uploads. The raw generated classes remain importable from hotdata.api.*.
+# uploads; hotdata.loads.DatabasesApi / ConnectionsApi retry table loads
+# while another load holds the table (409 RESOURCE_LOCKED). The raw
+# generated classes remain importable from hotdata.api.*.
 from hotdata.query import QueryApi as QueryApi  # noqa: E402,F811
 from hotdata.arrow import ResultsApi as ResultsApi  # noqa: E402,F811
 from hotdata.uploads import UploadsApi as UploadsApi  # noqa: E402,F811
+from hotdata.loads import DatabasesApi as DatabasesApi  # noqa: E402,F811
+from hotdata.loads import ConnectionsApi as ConnectionsApi  # noqa: E402,F811
+from hotdata.loads import (  # noqa: E402,F401
+    LoadRetryPolicy as LoadRetryPolicy,
+    TableLockedError as TableLockedError,
+)
+# Raised client-side (scripts/patch_workspace_guard.py) when a workspace-
+# scoped call has no workspace id configured.
+from hotdata._workspace import MissingWorkspaceError as MissingWorkspaceError  # noqa: E402,F401
 # The upload error hierarchy + option/helper types, so `from hotdata import
 # UploadError` / `StorageError` / ... works (they otherwise live only under
 # hotdata.uploads and are invisible to autocomplete on `hotdata.`).
@@ -350,4 +361,7 @@
     "UploadProgress",
     "UploadSource",
     "tqdm_progress",
+    "LoadRetryPolicy",
+    "TableLockedError",
+    "MissingWorkspaceError",
 ]

hotdata/_workspace.py

@@ -0,0 +1,84 @@
+"""Fail fast when a workspace-scoped request has no workspace to go to.
+
+Most operations are scoped to a workspace through the ``X-Workspace-Id``
+header (the ``WorkspaceId`` security scheme). The generated client simply
+skips a scheme that has no configured value, so a ``Configuration`` built
+without ``workspace_id`` sends the request anyway and the server answers with
+a bare 404 that says nothing about what was missing.
+
+:func:`ensure_workspace` runs before the request is sent (wired into
+``ApiClient.update_params_for_auth`` by ``scripts/patch_workspace_guard.py``)
+and raises :class:`MissingWorkspaceError` naming the constructor argument and
+the environment variable. Operations whose security does not list
+``WorkspaceId`` (workspace management itself) are unaffected, as are calls
+that supply the header explicitly or override auth per request.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Iterable, Mapping, Optional
+
+from hotdata.exceptions import ApiValueError
+
+WORKSPACE_SCHEME = "WorkspaceId"
+WORKSPACE_HEADER = "X-Workspace-Id"
+
+
+class MissingWorkspaceError(ApiValueError):
+    """Raised before sending a workspace-scoped request when no workspace id
+    is configured.
+
+    Pass ``workspace_id=`` to :class:`hotdata.Configuration`, set the
+    ``HOTDATA_WORKSPACE`` environment variable, or send the ``X-Workspace-Id``
+    header explicitly for the call.
+    """
+
+    def __init__(self, resource_path: str, method: str) -> None:
+        self.resource_path = resource_path
+        self.method = method
+        super().__init__(
+            f"{method} {resource_path} is scoped to a workspace but no workspace id "
+            "is configured. Pass workspace_id=... to hotdata.Configuration(...), "
+            "set the HOTDATA_WORKSPACE environment variable, or send the "
+            f"{WORKSPACE_HEADER} header explicitly (_headers=...)."
+        )
+
+
+def ensure_workspace(
+    configuration: Any,
+    auth_settings: Optional[Iterable[str]],
+    headers: Optional[Mapping[str, Any]],
+    resource_path: str,
+    method: str,
+    request_auth: Optional[Mapping[str, Any]] = None,
+) -> None:
+    """Raise :class:`MissingWorkspaceError` if the operation needs a workspace
+    id and none would be sent.
+
+    :param configuration: The client's :class:`hotdata.Configuration`.
+    :param auth_settings: The operation's security scheme names.
+    :param headers: Header params already assembled for the request.
+    :param request_auth: A per-request auth override; when given the
+        configuration is bypassed entirely, so the check is skipped.
+    """
+    if request_auth:
+        return
+    if not auth_settings or WORKSPACE_SCHEME not in auth_settings:
+        return
+    if headers and any(
+        str(name).lower() == WORKSPACE_HEADER.lower() for name in headers
+    ):
+        return
+    settings = configuration.auth_settings()
+    scheme = settings.get(WORKSPACE_SCHEME) if settings else None
+    if scheme and scheme.get("value"):
+        return
+    raise MissingWorkspaceError(resource_path, method)
+
+
+__all__ = [
+    "WORKSPACE_HEADER",
+    "WORKSPACE_SCHEME",
+    "MissingWorkspaceError",
+    "ensure_workspace",
+]

hotdata/api_client.py

@@ -29,6 +29,7 @@
 
 from hotdata.configuration import Configuration
 from hotdata.api_response import ApiResponse, T as ApiResponseT
+from hotdata._workspace import ensure_workspace
 from hotdata._useragent import USER_AGENT
 import hotdata.models
 from hotdata import rest
@@ -637,6 +638,19 @@ def update_params_for_auth(
         if not auth_settings:
             return
 
+        # Hand-applied (scripts/patch_workspace_guard.py): a workspace-scoped
+        # operation with no workspace id configured would otherwise go out
+        # without X-Workspace-Id and come back as a bare 404. Fail here with
+        # an error that says what is missing and how to supply it.
+        ensure_workspace(
+            self.configuration,
+            auth_settings,
+            headers,
+            resource_path,
+            method,
+            request_auth=request_auth,
+        )
+
         if request_auth:
             self._apply_auth_params(
                 headers,

hotdata/configuration.py

@@ -16,6 +16,7 @@
 import logging
 from logging import FileHandler
 import multiprocessing
+import os
 import sys
 from importlib.metadata import PackageNotFoundError, version as _pkg_version
 from typing import Any, ClassVar, Dict, List, Literal, Optional, TypedDict, Union
@@ -135,13 +136,16 @@ class HostSetting(TypedDict):
 class Configuration:
     """This class contains various settings of the API client.
 
-    :param host: Base url.
+    :param host: Base url. Defaults to the `HOTDATA_API_URL` environment
+      variable when set, else `https://api.hotdata.dev`.
     :param ignore_operation_servers
       Boolean to ignore operation servers for the API client.
       Config will use `host` as the base url regardless of the operation servers.
     :param api_key: Hotdata API key, sent as `Authorization: Bearer <key>`.
+      Defaults to the `HOTDATA_API_KEY` environment variable when not given.
     :param workspace_id: Public id of the target workspace, sent as
-      `X-Workspace-Id`.
+      `X-Workspace-Id`. Defaults to the `HOTDATA_WORKSPACE` environment
+      variable when not given.
     :param api_keys: Escape hatch for raw apiKey-scheme values, keyed by
       security scheme name. Prefer `workspace_id`.
     :param api_key_prefix: Dict to store API prefix (e.g. Bearer).
@@ -203,6 +207,14 @@ def __init__(
     ) -> None:
         """Constructor
         """
+        # Environment defaults, matching the Hotdata CLI: an explicit argument
+        # always wins; an unset or empty variable leaves the default in place.
+        if api_key is None:
+            api_key = os.environ.get("HOTDATA_API_KEY") or None
+        if workspace_id is None:
+            workspace_id = os.environ.get("HOTDATA_WORKSPACE") or None
+        if host is None:
+            host = os.environ.get("HOTDATA_API_URL") or None
         self._base_path = "https://api.hotdata.dev" if host is None else host
         """Default Base url
         """