feat: add the project administration and eval resources the CLI calls

.jscpd.json

@@ -14,6 +14,6 @@
   "mode": "weak",
   "kind": ["exact", "renamed"],
   "ignoreIdentifiers": true,
-  "threshold": 16.3,
+  "threshold": 16.1,
   "reporters": ["console"]
 }

README.md

@@ -511,6 +511,64 @@ caller who is not a project administrator only ever gets their own
 connections. The routes need `connections:read`, `connections:write` and
 `connections:delete`.
 
+## Administer a project
+
+These Control Plane namespaces need an org credential: an owner's or admin's
+session (`IntrospectionClient(cp_session=...)`) or an access token minted for
+one. A project API key carries none of their scopes. Pass `project=` (a slug
+or id) on project-scoped calls when the credential does not name one.
+
+| Namespace | What it manages |
+| --- | --- |
+| `client.oidc.me()` | Who the credential is, with the org's resolved feature flags |
+| `client.projects` | The org's projects, read by slug or id (`list`, `get`) |
+| `client.api_keys` | A project's API keys: `create` returns the plaintext once, then `acknowledge`; `delete(expires_in=...)` rotates |
+| `client.applications` | OAuth clients, with `.idps`, `.secrets` and `.keys` under each |
+| `client.credentials` / `client.variables` | Secrets and environment variables a project's runtimes read |
+| `client.integrations` | The org's own provider installs (org-wide, no `project`) |
+| `client.judges` | The judges a Recipe declares: read, and toggle `enabled` / `sample_rate` |
+
+```python
+client = IntrospectionClient(cp_session=os.environ["INTROSPECTION_CP_SESSION"])
+
+created = client.api_keys.create(project="support", name="ci")
+save(created.api_key.get_secret_value())  # a SecretStr: it never prints
+client.api_keys.acknowledge(created.id, project="support")
+
+client.credentials.create(name="OPENAI_API_KEY", secret="sk-...", project="support")
+```
+
+A secret the API returns once (an API key, a client secret) or takes on write
+(a credential, an IdP client secret, an integration's OAuth `code`) is a
+pydantic `SecretStr`, so `repr` and logging show `**********`.
+
+## Images and task commands
+
+`images` and `tasks.commands` are on the Data Plane, on the client and a
+Runner alike, for evaluation trials on a plane with evals enabled. An image is
+what an eval task's compose services run as; a command is `exec`, `upload` or
+`download` against an eval task's live sandbox.
+
+```python
+from introspection_sdk.schemas.images import MirrorImageSource
+from introspection_sdk.schemas.task_commands import ExecCommand
+
+image = client.images.create(
+    name="hello-mcp/mcp-server",
+    source=MirrorImageSource(reference="ghcr.io/acme/mcp@sha256:..."),
+)
+# Poll client.images.get(image.id) until status is "ready".
+
+task = client.tasks.create(prompt="Fix the failing test", kind="eval", commands=True)
+command = client.tasks.commands.create(str(task.task.id), ExecCommand(command=["pytest", "-q"]))
+command = client.tasks.commands.get(str(task.task.id), command.id, wait_seconds=25)
+print(command.status, command.exit_code, command.stdout)
+```
+
+`wait_seconds` holds the read until the command finishes; it is capped at
+`MAX_COMMAND_WAIT_SECONDS` (25, below the client's 30-second request timeout),
+so call `get` again to wait longer.
+
 ## Environment variables
 
 ```shell

examples/introspection_examples/api/project_admin.py

@@ -0,0 +1,89 @@
+"""Administer a project: who you are, its API keys and runtime settings.
+
+Reads the signed-in member, the org's projects, and a project's API keys,
+credentials, variables and judges; then, when INTROSPECTION_ADMIN_WRITE=1,
+mints an API key, saves its one-time plaintext, acknowledges it and revokes
+it again.
+
+These routes need an org credential: an owner's or admin's Control Plane
+session (or an access token minted for one). A project API key carries none
+of their scopes and gets a 403, which is why this example is not in
+``examples.yml``.
+
+Run with:
+    INTROSPECTION_CP_SESSION=...   # the intro_cp_session cookie value
+    INTROSPECTION_PROJECT=support  # project slug or id
+
+        uv run python -m introspection_examples.api.project_admin
+
+Optional env:
+    INTROSPECTION_ADMIN_WRITE   - 1 to also mint and revoke an API key
+    INTROSPECTION_BASE_API_URL  - CP REST API host (default https://api.introspection.dev)
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+from dotenv import load_dotenv
+from introspection_sdk import IntrospectionClient
+from introspection_sdk.schemas.lanes import Environment
+
+
+def main() -> None:
+    load_dotenv()
+    session = os.getenv("INTROSPECTION_CP_SESSION")
+    project = os.getenv("INTROSPECTION_PROJECT")
+    if not session or not project:
+        sys.exit(
+            "Set INTROSPECTION_CP_SESSION to an owner's Control Plane session "
+            "and INTROSPECTION_PROJECT to a project slug or id."
+        )
+
+    client = IntrospectionClient(cp_session=session)
+    try:
+        # #region docs:project-admin
+        me = client.oidc.me()
+        print(f"signed in as member {me.member_id} of org {me.org_id}")
+
+        for each in client.projects.list():
+            print(f"  project {each.slug or each.id}: {each.name}")
+
+        # `project` names the project on every project-scoped call, since a
+        # member session names none of its own.
+        for key in client.api_keys.list(project=project):
+            print(f"  key {key.key_display} {key.status} {key.environment}")
+        for credential in client.credentials.list(project=project):
+            print(
+                f"  credential {credential.name} {credential.secret_display}"
+            )
+        for variable in client.variables.list(project=project):
+            print(f"  variable {variable.name}={variable.value}")
+        for judge in client.judges.list(project=project):
+            print(f"  judge {judge.name} sample_rate={judge.sample_rate}")
+        # #endregion docs:project-admin
+
+        if os.getenv("INTROSPECTION_ADMIN_WRITE") == "1":
+            # #region docs:api-key-rotate
+            created = client.api_keys.create(
+                project=project,
+                name="example",
+                environment=Environment.DEVELOPMENT,
+            )
+            # A SecretStr: printing `created` shows **********. This is the
+            # only time the API returns the plaintext.
+            plaintext = (
+                created.api_key.get_secret_value() if created.api_key else ""
+            )
+            print(f"minted {created.key_display} ({len(plaintext)} chars)")
+            client.api_keys.acknowledge(created.id, project=project)
+            # `expires_in` keeps the key working that long, for a rotation.
+            client.api_keys.delete(created.id, project=project, expires_in=0)
+            # #endregion docs:api-key-rotate
+    finally:
+        client.shutdown()
+
+
+if __name__ == "__main__":
+    main()

introspection_sdk/_async/client.py

@@ -14,20 +14,29 @@
 from introspection_sdk.dev_target import client_headers
 from introspection_sdk.resources import (
     AsyncAnnotations,
+    AsyncApiKeys,
+    AsyncApplications,
     AsyncAutomations,
     AsyncConnectors,
+    AsyncCredentials,
     AsyncExperiments,
+    AsyncIntegrations,
+    AsyncJudges,
     AsyncMembers,
+    AsyncOidc,
+    AsyncProjects,
     AsyncRecipes,
     AsyncRepositories,
     AsyncRuntimes,
     AsyncTags,
+    AsyncVariables,
 )
 from introspection_sdk.runner_resources import (
     AsyncAppConnections,
     AsyncConversations,
     AsyncEvents,
     AsyncFiles,
+    AsyncImages,
     AsyncMetrics,
     AsyncShares,
     AsyncTasks,
@@ -88,6 +97,15 @@ class AsyncIntrospectionClient:
     metrics: AsyncMetrics
     shares: AsyncShares
     connections: AsyncAppConnections
+    images: AsyncImages
+    projects: AsyncProjects
+    api_keys: AsyncApiKeys
+    applications: AsyncApplications
+    credentials: AsyncCredentials
+    variables: AsyncVariables
+    integrations: AsyncIntegrations
+    judges: AsyncJudges
+    oidc: AsyncOidc
 
     def __init__(
         self,
@@ -147,6 +165,18 @@ def __init__(
         self.metrics = AsyncMetrics(self._dp_http)
         self.shares = AsyncShares(self._dp_http)
         self.connections = AsyncAppConnections(self._dp_http)
+        self.images = AsyncImages(self._dp_http)
+        # Project administration: each route needs an org credential (a
+        # member session or access token); a project API key carries none
+        # of their scopes.
+        self.projects = AsyncProjects(self._http)
+        self.api_keys = AsyncApiKeys(self._http)
+        self.applications = AsyncApplications(self._http)
+        self.credentials = AsyncCredentials(self._http)
+        self.variables = AsyncVariables(self._http)
+        self.integrations = AsyncIntegrations(self._http)
+        self.judges = AsyncJudges(self._http)
+        self.oidc = AsyncOidc(self._http)
 
     @classmethod
     async def from_service_account(

introspection_sdk/_async/protocols.py

@@ -44,6 +44,14 @@
 )
 from introspection_sdk.schemas.files import File, FileType
 from introspection_sdk.schemas.genai_span import GenAiSpan, GenAiSpanList
+from introspection_sdk.schemas.images import (
+    ContextImageSource,
+    Image,
+    ImageKind,
+    ImageProvenance,
+    ImageStatus,
+    MirrorImageSource,
+)
 from introspection_sdk.schemas.metrics import (
     MetricQueryRequest,
     MetricQueryResponse,
@@ -53,12 +61,20 @@
     ResourceShare,
     ShareResourceType,
 )
+from introspection_sdk.schemas.task_commands import (
+    DownloadCommand,
+    ExecCommand,
+    TaskCommand,
+    TaskCommandStatus,
+    UploadCommand,
+)
 from introspection_sdk.schemas.tasks import (
     Task,
     TaskCancelRequest,
     TaskCancelResponse,
     TaskCreateResponse,
     TaskFileRef,
+    TaskKind,
     TaskPrompt,
     TaskRepoRequest,
     TaskRun,
@@ -105,13 +121,47 @@ def stream(
     ) -> AsyncIterator[AGUIEvent]: ...
 
 
+@runtime_checkable
+class AsyncTaskCommandsResource(Protocol):
+    """Protocol of :class:`~introspection_sdk.runner_resources.tasks.AsyncTaskCommands`."""
+
+    async def create(
+        self,
+        task_id: str,
+        command: ExecCommand
+        | UploadCommand
+        | DownloadCommand
+        | dict[str, Any],
+    ) -> TaskCommand: ...
+
+    def list(
+        self,
+        task_id: str,
+        *,
+        status: TaskCommandStatus | str | None = None,
+        limit: int | None = None,
+        next: str | None = None,
+    ) -> AsyncPager[TaskCommand, Paginated[TaskCommand]]: ...
+
+    async def get(
+        self,
+        task_id: str,
+        command_id: str | UUID,
+        *,
+        wait_seconds: int | None = None,
+    ) -> TaskCommand: ...
+
+
 @runtime_checkable
 class AsyncTasksResource(Protocol):
     """Protocol of :class:`~introspection_sdk.runner_resources.tasks.AsyncTasks`."""
 
     @property
     def runs(self) -> AsyncTaskRunsResource: ...
 
+    @property
+    def commands(self) -> AsyncTaskCommandsResource: ...
+
     def list(
         self,
         *,
@@ -137,6 +187,8 @@ async def create(
         fork_share_id: str | None = None,
         files: builtins.list[TaskFileRef | dict[str, Any]] | None = None,
         tags: builtins.list[str] | None = None,
+        kind: TaskKind | str | None = None,
+        commands: bool | None = None,
     ) -> TaskCreateResponse: ...
 
     async def get(self, task_id: str) -> Task: ...
@@ -596,6 +648,35 @@ async def get(self, connection_id: UUID | str) -> Connection: ...
     async def delete(self, connection_id: UUID | str) -> None: ...
 
 
+@runtime_checkable
+class AsyncImagesResource(Protocol):
+    """Protocol of :class:`~introspection_sdk.runner_resources.images.AsyncImages`."""
+
+    def list(
+        self,
+        *,
+        kind: ImageKind | str | None = None,
+        status: ImageStatus | str | None = None,
+        name: str | None = None,
+        runtime_id: UUID | None = None,
+        limit: int | None = None,
+        next: str | None = None,
+    ) -> AsyncPager[Image, Paginated[Image]]: ...
+
+    async def get(self, image_id: UUID | str) -> Image: ...
+
+    async def create(
+        self,
+        *,
+        name: str,
+        source: ContextImageSource | MirrorImageSource | dict[str, Any],
+        provenance: ImageProvenance | dict[str, Any] | None = None,
+        scan: bool | None = None,
+    ) -> Image: ...
+
+    async def delete(self, image_id: UUID | str) -> None: ...
+
+
 @runtime_checkable
 class AsyncDataPlaneResources(Protocol):
     """The data-plane namespaces :class:`~introspection_sdk.AsyncIntrospectionClient`
@@ -624,3 +705,6 @@ def automations(self) -> AsyncAutomationsResource: ...
 
     @property
     def connections(self) -> AsyncConnectionsResource: ...
+
+    @property
+    def images(self) -> AsyncImagesResource: ...