diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b78d886 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +# Only the staged binaries are needed in the build context - without this the +# context drags in bin/ and dist/ (hundreds of MB of zips and bundles). +* +!.oci-stage diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dfd82a6..7a0a07c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -97,3 +97,127 @@ jobs: - name: Smoke test (deterministic MCP handshake) run: python scripts/smoke-test.py dist/stackql-mcp-windows-x64.mcpb + + # OCI image: built from the release zips (which exist as soon as the + # upstream release does, so this runs at PR time), smoke-tested in-container. + # Pushing happens in publish.yml, gated on Docker Hub secrets. + oci: + name: oci (build + smoke) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Build image + run: make oci VERSION=${{ inputs.version }} + + - name: Smoke test (deterministic MCP handshake) + run: python3 scripts/smoke-test.py --docker stackql/stackql-mcp:${{ inputs.version }} + + # npm wrapper: the real platforms.json pins the sha256 of the PUBLISHED + # .mcpb assets, which do not exist yet at PR time for a new version (this + # repo publishes them at tag time). CI therefore tests the wrapper's + # extract/cache/spawn logic against a locally built bundle via the + # STACKQL_MCP_BUNDLE override, with placeholder pins. The download+verify + # path is exercised by 'make npm-manifest && npm pack' after publish. + npm-wrapper: + name: npm wrapper (smoke) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Build local bundle for the wrapper to consume + run: make one TARGET=linux-x64 VERSION=${{ inputs.version }} + + - name: Render placeholder manifest + run: | + zeros="0000000000000000000000000000000000000000000000000000000000000000" + cat > npm/platforms.json < pypi/src/stackql_mcp_server/platforms.json < /dev/null && echo "mcp-config is valid JSON" + python3 - <<'EOF' + import importlib.util, json, os + cfg = json.loads(os.environ["MCP_CONFIG"]) + server = cfg["mcpServers"]["stackql"] + spec = importlib.util.spec_from_file_location("smoke", "scripts/smoke-test.py") + smoke = importlib.util.module_from_spec(spec) + spec.loader.exec_module(smoke) + smoke.run_command([server["command"], *server["args"]]) + EOF diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2ef7c0c..76cc725 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -106,3 +106,70 @@ jobs: env: GH_TOKEN: ${{ secrets.STACKQL_RELEASE_TOKEN }} run: make publish VERSION=${{ needs.verify-tag.outputs.version }} + + # Multi-arch OCI image push. Soft-skips unless DOCKERHUB_USERNAME and + # DOCKERHUB_TOKEN secrets are set (token needs push rights on + # docker.io/stackql/stackql-mcp). + oci-publish: + name: push OCI image (optional) + needs: [verify-tag, build] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Build and push multi-arch image + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + run: | + if [ -z "$DOCKERHUB_USERNAME" ] || [ -z "$DOCKERHUB_TOKEN" ]; then + echo "OCI push skipped (set DOCKERHUB_USERNAME + DOCKERHUB_TOKEN secrets)" + exit 0 + fi + echo "$DOCKERHUB_TOKEN" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin + docker buildx create --use + make oci-push VERSION=${{ needs.verify-tag.outputs.version }} + + # Render the npm manifest from the freshly published .sha256 files and build + # the tarball as a run artifact. Publishing to npmjs stays MANUAL (2FA): + # download the artifact (or run 'make npm-pack' locally), then + # 'cd npm && npm publish --access public'. + npm-tarball: + name: npm tarball (manual publish artifact) + needs: [verify-tag, publish] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Render manifest from published assets and pack + run: make npm-pack VERSION=${{ needs.verify-tag.outputs.version }} + + - name: Upload tarball artifact + uses: actions/upload-artifact@v7 + with: + name: npm-package + path: npm/*.tgz + if-no-files-found: error + + # Render the pypi manifest from the freshly published .sha256 files and + # build sdist + wheel as a run artifact. Publishing to PyPI stays MANUAL + # (2FA / token): download the artifact (or run 'make pypi-build' locally), + # then 'python -m twine upload pypi/dist/*'. + pypi-dist: + name: pypi dist (manual publish artifact) + needs: [verify-tag, publish] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Render manifest from published assets and build + run: | + python3 -m pip install --quiet build + make pypi-build VERSION=${{ needs.verify-tag.outputs.version }} + + - name: Upload dist artifact + uses: actions/upload-artifact@v7 + with: + name: pypi-package + path: pypi/dist/* + if-no-files-found: error diff --git a/.gitignore b/.gitignore index b09b5e0..76ad476 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,11 @@ node_modules/ tmp/ /.stackql /.tools +/.oci-stage +npm/platforms.json +npm/package-lock.json +npm/*.tgz +pypi/dist/ +pypi/src/stackql_mcp_server/platforms.json +pypi/src/*.egg-info/ +/.venv* diff --git a/CLAUDE.md b/CLAUDE.md index d800366..648e9ed 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,230 +1,259 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## What this repo is - -A standalone, scripted post-release step that packages the StackQL MCP server into per-platform [MCPB](https://github.com/anthropics/mcpb) bundles (`.mcpb`) for distribution and listing on the official MCP Registry. - -This repo does NOT build or sign the stackql binaries - that happens upstream in the normal stackql build/signing process. Here you drop the already-signed release artefacts (per-arch zips and the notarised macOS `.pkg`) into `bin/`, run one script, and get signed `.mcpb` bundles plus checksums in `dist/` to attach to the matching GitHub release. - -The server packed into each bundle is the `stackql` binary itself, launched as `stackql mcp --mcp.server.type=stdio` (see [manifest/manifest.template.json](manifest/manifest.template.json)). The `--mcp.server.type=stdio` flag is required - without it the MCP server does not produce JSON-RPC on stdout. The separate `stackql_mcp_client` binary is a test client and is NOT packaged. - -## Common commands - -A [Makefile](Makefile) wraps `scripts/package.sh` for the common flows. The script is still the source of truth; `make` is convenience. - -`VERSION` defaults to the `stackql_release` value pinned in [release.yaml](release.yaml) (leading `v` stripped), so plain `make all` builds the pinned release. Passing `VERSION=X.Y.Z` overrides it. `release.yaml` is also what CI builds on PRs and what a pushed tag must match to publish (see "Release flow" below). - -One-shot from a clean checkout - downloads the release artefacts from `https://github.com/stackql/stackql/releases/download/v/...` into `bin/`, then builds every available bundle: - -```bash -make all VERSION=X.Y.Z -# 'make VERSION=X.Y.Z' is equivalent ('all' is the default target) -# 'make' alone uses the version pinned in release.yaml -``` - -Just download (skip packaging): - -```bash -make download VERSION=X.Y.Z -``` - -Just package whatever is already in `bin/` (skip downloading): - -```bash -make package VERSION=X.Y.Z -# or call the script directly: -./scripts/package.sh --version X.Y.Z -``` - -Build a single target. Two variants: - -```bash -# Download just that target's source artefact and build only that bundle. -# Use this on a Mac to do the darwin slice in the two-machine release flow. -make one TARGET=darwin-universal VERSION=X.Y.Z -make one TARGET=linux-x64 VERSION=X.Y.Z - -# Build from already-present artefacts in bin/ (temporarily hides the -# others under bin/.hidden/ and restores them after). -make linux-x64 VERSION=X.Y.Z -make linux-arm64 VERSION=X.Y.Z -make windows-x64 VERSION=X.Y.Z -make darwin-universal VERSION=X.Y.Z -``` - -Self-signed bundles (testing only - production envelope signing is not currently wired up; see "Trust model" below): - -```bash -make signed VERSION=X.Y.Z -``` - -Upload everything in `dist/` to the matching `stackql/stackql` release (requires `gh auth login` with `contents:write` on `stackql/stackql`; idempotent via `--clobber`): - -```bash -make publish VERSION=X.Y.Z -``` - -Wipe outputs / inputs: - -```bash -make clean # remove dist/*.mcpb and *.sha256 -make clean-bin # remove downloaded artefacts from bin/ -``` - -Show what is currently in the drop-zone: - -```bash -make list -``` - -## Release flow - -The primary flow is GitHub Actions; the two-machine local flow below remains a supported fallback. The darwin target needs `pkgutil`, so CI builds it on a `macos-latest` runner. - -### CI flow (GitHub Actions) - -Three workflows in [.github/workflows/](.github/workflows/): - -- **[build.yml](.github/workflows/build.yml)** - reusable. Builds each bundle with `make one TARGET= VERSION=` on a runner that can execute the embedded binary (`ubuntu-latest`, `ubuntu-24.04-arm`, `macos-latest`), runs `scripts/smoke-test.py` against it, and uploads `dist/` artefacts. The windows-x64 bundle builds on ubuntu (packaging has no Windows dependency) and is smoke-tested on `windows-latest` from the artifact. The Gemini agent check runs on linux-x64 only and soft-skips without `GEMINI_API_KEY`. -- **[ci.yml](.github/workflows/ci.yml)** - on PRs to main and pushes to main. Reads `stackql_release` from [release.yaml](release.yaml) and calls build.yml. Nothing is published. -- **[publish.yml](.github/workflows/publish.yml)** - on pushing a `v*` tag. Fails fast if the tag does not exactly match `stackql_release` in release.yaml, rebuilds and smoke-tests everything, optionally envelope-signs the bundles (runs `make sign` when the `MCPB_SIGNING_CERT`/`MCPB_SIGNING_KEY` PEM-content secrets are set; soft-skips otherwise), then runs `make publish` to upload all bundles + `.sha256` files to the matching `stackql/stackql` release. Requires the `STACKQL_RELEASE_TOKEN` repo secret (fine-grained PAT with `contents:write` on `stackql/stackql` - the default `GITHUB_TOKEN` cannot upload cross-repo). Signing happens before checksumming: the signature is appended to the bundle bytes, so `make sign` regenerates each `.sha256`. `mcpb verify` is broken in the current upstream CLI (node-forge cannot verify PKCS#7); the scripts treat it as advisory and assert the appended `MCPB_SIG_END` block instead. - -The release sequence: upstream `stackql/stackql` release publishes the core assets -> raise a PR here bumping `stackql_release` in release.yaml (CI proves the bundles build and pass smoke tests against the real release assets) -> merge -> push the matching tag (e.g. `v0.10.500`) -> publish.yml attaches the `.mcpb` assets to the upstream release. - -### Fallback: the two-machine local flow - -**Machine A (your workstation, any OS with bash + node + unzip):** - -```bash -make all VERSION=X.Y.Z # downloads release artefacts, builds linux-x64, - # linux-arm64, windows-x64. Darwin skips with a - # 'pkgutil not found' notice. -python scripts/smoke-test.py dist/stackql-mcp-linux-x64.mcpb # gate -make publish VERSION=X.Y.Z # uploads the 3 bundles + .sha256s to the - # stackql/stackql release matching v -``` - -**Machine B (a Mac - MacInCloud is the typical case):** - -```bash -git clone https://github.com/stackql/stackql-mcpb-packaging -cd stackql-mcpb-packaging -make one TARGET=darwin-universal VERSION=X.Y.Z # downloads only the .pkg, - # extracts the universal - # binary, builds 1 bundle -python scripts/smoke-test.py dist/stackql-mcp-darwin-universal.mcpb -make publish VERSION=X.Y.Z # uploads just that one bundle + sha -``` - -Each machine runs `gh auth login` once with a token that has `contents:write` on `stackql/stackql`. `make publish` uses `gh release upload --clobber`, so it is idempotent and the order between the two machines does not matter. Re-running either step is safe. - -The Mac machine only needs Node.js (for `mcpb` via `npx`) on top of the default macOS toolchain - `make`, `curl`, `unzip`, `pkgutil`, `shasum`, `find` are all preinstalled. - -### Smoke tests - -Two layers, both in [scripts/](scripts/): - -- **[scripts/smoke-test.py](scripts/smoke-test.py)** - deterministic gate. Extracts the `.mcpb`, spawns `stackql mcp --mcp.server.type=stdio --auth='{"github":{"type":"null_auth"}}'`, runs the JSON-RPC handshake, asserts `tools/list` contains `pull_provider`/`list_services`/`list_providers`, calls `pull_provider` for `github`, then `list_services` and confirms real github services come back. Stdlib only. Run before `make publish`: - - ```bash - python scripts/smoke-test.py dist/stackql-mcp-linux-x64.mcpb - ``` - -- **[scripts/gemini-smoke.py](scripts/gemini-smoke.py)** - optional agent check using Gemini Flash. Exposes the MCP tools to Gemini via function calling and asks it to pull github and list services. Skips with exit 0 if `GEMINI_API_KEY` is not set; on failure prints `WARN:` and exits 0. Stdlib only - calls `generativelanguage.googleapis.com` directly via `urllib`. `GEMINI_MODEL` defaults to `gemini-2.0-flash`. - -Both scripts use the `github` provider in `null_auth` mode so no credentials are needed - they hit the public github registry endpoints. - -## Trust model - -The end goal is signed, verifiable, functional MCP binary assets distributed through trusted marketplaces. Today the layers are: - -1. **Mach-O / Authenticode signatures on the embedded binary** - applied upstream during the stackql release build. Windows: Authenticode-signed `stackql.exe`. macOS: Developer ID Application signature embedded in the universal `stackql` binary inside the `.pkg`, plus Apple notarisation keyed to the binary's cdhash. Linux: no platform-level signing, by convention. -2. **SHA-256 on the bundle envelope** - written by `package.sh` next to every `.mcpb`. Published with the bundle and pinned in the official MCP Registry `server.json`. Anyone installing the bundle can verify the bytes. -3. **MCPB envelope signature (`mcpb sign`)** - wired but inactive until signing material is configured. `make sign` (scripts/sign.sh) signs `dist/*.mcpb` in place and regenerates checksums; the publish workflow calls it and soft-skips unless the `MCPB_SIGNING_CERT`/`MCPB_SIGNING_KEY` secrets are set. The same `MCPB_SELF_SIGN`/`MCPB_SIGN_CERT`/`MCPB_SIGN_KEY`/`MCPB_SIGN_INTERMEDIATES` hooks remain in `package.sh` for sign-at-build. -4. **Anthropic Desktop Extensions directory listing** - the editorial "vetted by Claude" signal that users see in Claude Desktop's Browse Extensions UI. Submission is via the review form at `claude.com/docs/connectors/building/submission`; requirements (privacy policy, logo, screenshots) are in [listings.md](listings.md). -5. **Official MCP Registry entry** - canonical metadata pointing at the GitHub release assets and pinning their SHA-256. - -The notarised `.pkg` does the load-bearing trust work for macOS users: Gatekeeper validates the cdhash online when the bundled binary launches, so the binary inside the `.mcpb` is the same trusted binary users get from the `.pkg` installer. The unsigned `.mcpb` envelope is a Claude Desktop UI signal, not a Gatekeeper signal - addressing it requires either a self-signed cert (low value) or a real code-signing cert held in an HSM (the production answer). Until then, the registry SHA-256 plus the embedded platform signatures are what marketplaces verify against. - -Self-signed bundle (testing only): - -```bash -MCPB_SELF_SIGN=true ./scripts/package.sh --version X.Y.Z -``` - -Production-signed bundle: - -```bash -MCPB_SIGN_CERT=cert.pem \ -MCPB_SIGN_KEY=key.pem \ -MCPB_SIGN_INTERMEDIATES="intermediate-ca.pem root-ca.pem" \ -./scripts/package.sh --version X.Y.Z -``` - -`MCPB_SIGN_INTERMEDIATES` is optional and space-separated. Bundle signing is OFF by default. When unset, the script prints a notice and skips. - -The script invokes `mcpb` if on PATH, otherwise falls back to `npx --yes @anthropic-ai/mcpb`. - -## Bin drop-zone layout (required before running package.sh) - -`bin/` is gitignored except for its `README.md` and `.gitignore`. `package.sh` reads the release artefacts directly - no manual extraction. Expected files at the root of `bin/`: - -``` -bin/ - stackql_linux_amd64.zip # contains stackql - stackql_linux_arm64.zip # contains stackql - stackql_windows_amd64.zip # contains stackql.exe (Authenticode-signed upstream) - stackql_darwin_multiarch.pkg # notarised .pkg, universal binary inside -``` - -The darwin glob is `stackql_darwin*.pkg`, so any suffix works. Any target whose source artefact is absent is skipped with a notice - partial drops produce partial bundle sets. - -A legacy fallback layout (pre-extracted binaries at `bin//stackql[.exe]` and `bin/darwin/*.pkg`) is also accepted. The release-artefact layout takes precedence when both are present. - -## Architecture and flow - -Single bash script ([scripts/package.sh](scripts/package.sh)) drives everything. For each target: - -1. Stage a temp dir with `server/` and a per-target `manifest.json` rendered from [manifest/manifest.template.json](manifest/manifest.template.json) by `sed`-substituting `__VERSION__` and `__BINARY_NAME__`. -2. `mcpb validate` the manifest, then `mcpb pack` the staging dir into `dist/stackql-mcp-