diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dc738cb805..d621f9825e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -70,6 +70,9 @@ jobs: - name: Run unit tests with coverage run: > nix develop --command just test-coverage + - name: Validate independent fctl plugin + run: > + nix develop --command just test-fctl-plugin build-fctl-plugin - name: Run internal tests with optional features and coverage run: > nix develop --command just test-internal-coverage diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6ba5118338..e1d6da2b62 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,6 +10,8 @@ permissions: jobs: GoReleaser: + env: + PLUGIN_REVISION: "1" runs-on: namespace-profile-linux-amd64-4vcpu steps: - uses: namespacelabs/nscloud-checkout-action@445c25d7009680597d73eb03c4e1cd5be522ed73 # v9 @@ -18,6 +20,9 @@ jobs: dissociate: true - name: Setup Env uses: ./.github/actions/default + - name: Validate fctl plugin before co-release + run: > + nix develop --command just test-fctl-plugin - name: Set up Docker Buildx uses: namespacelabs/nscloud-setup-buildx-action@d059ed7184f0bc7c8b27e8810cea153d02bcc6dd # v0.0.23 - name: Login to GitHub Container Registry @@ -31,6 +36,10 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} + - name: Publish Ledger fctl plugin and release catalogue + run: nix develop --command just publish-fctl-plugin-release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build and push operator image run: > nix develop --command just operator-docker-build ${{ github.ref_name }} diff --git a/.goreleaser.yml b/.goreleaser.yml index c94fbd972f..31fdb416a6 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -8,6 +8,7 @@ project_name: ledger # `formancehq/ledger` published image. env: - IMAGE_REPOSITORY={{ envOrDefault "IMAGE_REPOSITORY" "ghcr.io/formancehq/ledger" }} +- PLUGIN_REVISION={{ envOrDefault "PLUGIN_REVISION" "1" }} builds: - binary: ledger-server @@ -66,6 +67,26 @@ builds: - amd64 - arm64 +# The product owns its HTTP fctl commands and co-releases their executable. +- binary: fctl-plugin-ledger + id: fctl-plugin-ledger + dir: misc/fctl-plugin + main: ./cmd/fctl-plugin-ledger + # Keep -ldflags in Go build metadata for publication identity checks. + ldflags: + - -s -w + - -X main.serviceVersion={{ .Version }} + - -X main.revision={{ .Env.PLUGIN_REVISION }} + env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + - windows + goarch: + - amd64 + - arm64 + nightly: tag_name: nightly publish_release: true @@ -98,6 +119,19 @@ archives: - zip name_template: "{{ .ProjectName }}_{{ if not .IsNightly }}v{{ end }}{{ .Version }}_{{ .Os }}-{{ .Arch }}" wrap_in_directory: true +- id: fctl-plugin-ledger + ids: + - fctl-plugin-ledger + formats: + - tar.gz + format_overrides: + - goos: windows + formats: + - zip + name_template: "fctl-plugin-ledger_v{{ .Version }}_r{{ .Env.PLUGIN_REVISION }}_{{ .Os }}-{{ .Arch }}" + files: + - src: misc/fctl-plugin/README.md + dst: README.md checksum: name_template: "checksums.txt" diff --git a/docs/ops/cli.md b/docs/ops/cli.md index 68cacbc088..1ed0df8fe6 100644 --- a/docs/ops/cli.md +++ b/docs/ops/cli.md @@ -2,6 +2,14 @@ `ledgerctl` is the command-line client for interacting with Ledger v3 servers via gRPC. +Ledger also owns an independent HTTP command plugin for fctl v4. The +`fctl-plugin-ledger` executable is built from the separate `misc/fctl-plugin/` Go +module and co-released with the Ledger service for Linux, macOS, and Windows +on amd64 and arm64. fctl supplies profiles, authentication, interactive forms, +HTTP diagnostics, and output styling. The plugin supplies the Ledger command +manifest and HTTP operations. See the [fctl plugin guide](../../misc/fctl-plugin/README.md) +for local builds, service versions, plugin revisions, and distribution inputs. + ## Getting Started ![Getting Started](../../misc/demo/demo_getting_started.gif) diff --git a/docs/technical/architecture/subsystems/api/README.md b/docs/technical/architecture/subsystems/api/README.md index d8931a910c..5933669da4 100644 --- a/docs/technical/architecture/subsystems/api/README.md +++ b/docs/technical/architecture/subsystems/api/README.md @@ -12,6 +12,7 @@ Client-facing transport layers (`internal/adapter/grpc`, `internal/adapter/http` | [http-api.md](http-api.md) | HTTP endpoint lifecycle and failure shutdown, REST API endpoints, single-decoding metadata key paths, response formats, opt-in string monetary amounts, exact metadata number decoding, ledger-log JSON output and discriminators, error handling, and the `apierr`/`grpcerr` error boundary crossed by a forwarded write. | | [auth.md](auth.md) | Client JWT authentication (OIDC + Ed25519), scope-based authorization, and the Raft inter-node cluster-secret auth layer. | | [sensitive-projection.md](sensitive-projection.md) | Detached, descriptor-driven secret projection and its ownership boundary. | +| [fctl-plugin.md](fctl-plugin.md) | Product-owned HTTP command module, external SDK broker, co-release identity, and immutable OCI publication. | Decoded leader errors retain their exact gRPC status through routing wrappers and streaming cursors. See [forwarding and cancellation](http-api.md#forwarded-writes-and-the-transport-seam) diff --git a/docs/technical/architecture/subsystems/api/fctl-plugin.md b/docs/technical/architecture/subsystems/api/fctl-plugin.md new file mode 100644 index 0000000000..703042d690 --- /dev/null +++ b/docs/technical/architecture/subsystems/api/fctl-plugin.md @@ -0,0 +1,60 @@ +# Product-owned fctl plugin + +With Ledger removed from the fctl binary, users need an installable exact-version +command provider. Ledger co-releases that provider so its API owner can keep +commands aligned with the service without pulling server or host CLI code into +the module. Auth PR #162 establishes the same release/distribution contract. + +Ledger owns its HTTP command package in the separate `misc/fctl-plugin/` Go module. +Its sole direct dependency is the public fctl plugin SDK. It imports neither +Ledger service internals nor the fctl core, UI, profiles, or authentication. + +The package implements the SDK's `GetManifest` and `Execute` contract. The +manifest declares command arguments, flags, confirmations, and forms. The same +package can run embedded or through the standalone `fctl-plugin-ledger` +executable. The external entry point calls `transport.Serve` with the factory +that accepts the host-provided HTTP client. + +In external mode, the SDK broker returns each HTTP request to the fctl host. +The host applies its authentication, endpoint restrictions, transport, and +diagnostics. Exact JSON bytes and partial bulk responses stay on the public +SDK boundary. The executable has no Ledger service gRPC contract dependency; +the plugin SDK protocol version is distinct from `pkg/grpcprotocol.Version`. + +The existing command contract includes shape validation before dispatch, raw JSON +numbers, page cursors and partial bulk errors. Validation rejects malformed input +without executing a mutation; state-dependent business rules remain server-owned. + +Each plugin release names an exact Ledger service version and a positive +plugin revision. A CLI-only fix increments the revision for that service +version. It does not change the service version in the SDK manifest. The +host's catalogue maps this identity and platform to a digest-addressed native +executable. The host validates the running manifest against the locked one. + +GoReleaser builds the six supported native platforms in the Ledger release +pipeline. The publisher reads the resulting GoReleaser artifact list and the +SDK manifest exported by the native executable. It validates each binary's Go +entry point, target platform, and injected service version and plugin revision, +computes its SHA-256 checksum, and uploads +one raw executable layer per OCI artifact through the Nix-pinned ORAS tool. +Paths from the artifact list are opened through `os.Root`; traversal and +symlink escapes cannot select a file outside the build root. + +Publication builds retain Go linker metadata. They do not use `-trimpath`, +which removes the recorded `-ldflags` needed to verify foreign-platform +version and revision without executing the binary. The publisher checks every +platform before calling ORAS. A stale manifest or a mixed set of product +versions fails preflight and produces neither an upload nor a catalogue. + +The publisher creates private staging files before uploading. It emits a +catalogue only after all six uploads succeed. If an upload fails, earlier +artifacts can remain in the registry, but no complete catalogue is emitted. +The canonical Just recipe replaces the prepared catalogue only on success. +The local layout recipe exercises the same ORAS serialization without any +registry upload. See the [build and publication guide](../../../../../misc/fctl-plugin/README.md). + +Tagged release CI invokes the publisher after GoReleaser, verifies the exact +service tag as well as every platform identity, uses the job token for GHCR, and +attaches `fctl-plugin-catalogue.json` after all uploads. The first publication +requires Public GHCR visibility and anonymous checksum verification. Promotion +into `formancehq/fctl-plugin-registry` is a separate reviewed operation. diff --git a/docs/technical/contributing/getting-started.md b/docs/technical/contributing/getting-started.md index 0e97d687a3..31c3eae6d7 100644 --- a/docs/technical/contributing/getting-started.md +++ b/docs/technical/contributing/getting-started.md @@ -43,6 +43,7 @@ ledger/ ├── pkg/ # Public packages (actions/, scenario/, testserver/) ├── tests/ # Test suites (e2e/, scenarios/, antithesis/, perf/, schemathesis/) ├── misc/ +│ ├── fctl-plugin/ # Independent HTTP command plugin for fctl v4 (own Go module) │ ├── proto/ # Protocol Buffer definitions │ ├── demo/ # VHS tape files for CLI demos │ ├── numscript/examples/ # Numscript examples diff --git a/flake.nix b/flake.nix index 994d2e261c..f6b7b46efd 100644 --- a/flake.nix +++ b/flake.nix @@ -82,6 +82,8 @@ acli go_1_27 ffmpeg + gh + coreutils ginkgo gomarkdoc go-jsonnet @@ -92,6 +94,7 @@ k6 kubernetes-helm nodejs_22 + oras python314 trufflehog uv diff --git a/justfile b/justfile index 01852fe11e..6decd40f51 100644 --- a/justfile +++ b/justfile @@ -30,6 +30,8 @@ lint: golangci-lint run --fix --build-tags it,local,enable_antithesis_sdk,{{all_tags}} --timeout 5m echo "==> golangci-lint (operator)" (cd misc/operator && golangci-lint run --fix --timeout 5m) + echo "==> golangci-lint (fctl plugin)" + (cd misc/fctl-plugin && golangci-lint run --fix --build-tags it --timeout 5m) echo "==> golangci-lint (model workload)" (cd tests/antithesis/workload && golangci-lint run --fix --build-tags enable_antithesis_sdk --timeout 5m) @@ -40,6 +42,8 @@ tidy: go mod tidy echo "==> go mod tidy (operator)" (cd misc/operator && go mod tidy) + echo "==> go mod tidy (fctl plugin)" + (cd misc/fctl-plugin && go mod tidy) echo "==> go mod tidy (model workload)" (cd tests/antithesis/workload && go mod tidy) @@ -68,6 +72,45 @@ build-full: build-client: go build -o ./build/ledgerctl ./cmd/ledgerctl +# Build the HTTP fctl plugin, independently of the Ledger service module. +build-fctl-plugin version="dev" revision="1": + cd misc/fctl-plugin && CGO_ENABLED=0 go build -ldflags '-X main.serviceVersion={{version}} -X main.revision={{revision}}' -o ../../build/fctl-plugin-ledger ./cmd/fctl-plugin-ledger + +test-fctl-plugin: + #!/usr/bin/env bash + set -euo pipefail + # The SDK uses Unix sockets; the canonical validator's nested TMPDIR can + # exceed macOS sockaddr_un.sun_path. Keep real IPC tests in a short root. + fixture_tmp="$(mktemp -d /tmp/fctl-plugin.XXXXXX)" + trap 'rm -rf "$fixture_tmp"' EXIT + export TMPDIR="$fixture_tmp" + cd misc/fctl-plugin + go test -race -tags it ./... -timeout 3m + +# Prepare six native platform archives and checksums without publishing. +package-fctl-plugin version="dev" revision="1": + cd misc/fctl-plugin && LEDGER_SERVICE_VERSION={{version}} PLUGIN_REVISION={{revision}} goreleaser release --snapshot --clean --skip=publish + +# Export the SDK manifest from the previously built native executable. +fctl-plugin-manifest: + ./build/fctl-plugin-ledger --manifest > ./build/fctl-plugin-manifest.json + +# Publish previously packaged raw binaries; advertise the catalogue only on success. +publish-fctl-plugin registry="https://ghcr.io" repository="formancehq/fctl-plugin-ledger" revision="1": + #!/usr/bin/env bash + set -euo pipefail + cd misc/fctl-plugin + go run ./cmd/fctl-plugin-publish --service-version "$(jq -er .version ../../build/fctl-plugin-manifest.json)" --manifest ../../build/fctl-plugin-manifest.json --artifacts ../../build/fctl-plugin/artifacts.json --source-root ../../build/fctl-plugin --registry '{{registry}}' --repository '{{repository}}' --revision '{{revision}}' > ../../build/fctl-plugin-catalogue.json.tmp + mv ../../build/fctl-plugin-catalogue.json.tmp ../../build/fctl-plugin-catalogue.json + +# Exercise real ORAS artifact creation locally without accessing any registry. +fctl-plugin-oci-layout revision="1": + #!/usr/bin/env bash + set -euo pipefail + cd misc/fctl-plugin + go run ./cmd/fctl-plugin-publish --service-version "$(jq -er .version ../../build/fctl-plugin-manifest.json)" --manifest ../../build/fctl-plugin-manifest.json --artifacts ../../build/fctl-plugin/artifacts.json --source-root ../../build/fctl-plugin --layout ../../build/fctl-plugin-oci --revision '{{revision}}' > ../../build/fctl-plugin-catalogue.json.tmp + mv ../../build/fctl-plugin-catalogue.json.tmp ../../build/fctl-plugin-catalogue.json + # Run the application locally (single node) run: go run . run --node-id 1 --cluster-id local-dev --bootstrap --bind-addr 127.0.0.1:7777 --grpc-port 8888 --wal-dir ./wal/node-1 --data-dir ./data/node-1 @@ -548,3 +591,54 @@ test-sdk-bigint: npm --prefix "$sdk_dir" ci --ignore-scripts --no-audit --no-fund (cd "$sdk_dir" && node node_modules/typescript/bin/tsc) LEDGER_SDK_TEST="$sdk_dir/esm/ledger-bigint-test.js" go test -tags sdk ./internal/adapter/http -run ^TestBigintOperations$ -count=1 -timeout=2m + +# Release publication uses the native executable from this exact GoReleaser run. +publish-fctl-plugin-release: + #!/usr/bin/env bash + set -euo pipefail + : "${GITHUB_REF_NAME:?release tag is required}" + : "${GITHUB_TOKEN:?GitHub Actions token is required}" + : "${GITHUB_ACTOR:?publishing actor is required}" + [[ "$GITHUB_REF_NAME" == v* ]] || { echo 'Expected a Ledger v-prefixed release tag' >&2; exit 1; } + version="${GITHUB_REF_NAME#v}" + revision="${PLUGIN_REVISION:-1}" + native="$(jq -er '[.[] | select(.type == "Binary" and .extra.ID == "fctl-plugin-ledger" and .goos == "linux" and .goarch == "amd64")] | if length == 1 then .[0].path else error("expected one native Ledger plugin") end' dist/artifacts.json)" + : "${GITHUB_REPOSITORY:?release repository is required}" + release_root="$PWD" + auth_dir="$(mktemp -d)" + trap 'rm -rf "$auth_dir"; rm -f "$release_root/dist/fctl-plugin-catalogue.json.tmp" "$release_root/dist/fctl-plugin-manifest.json.tmp"' EXIT + export DOCKER_CONFIG="$auth_dir" + "$native" --manifest > dist/fctl-plugin-manifest.json.tmp + printf '%s' "$GITHUB_TOKEN" | oras login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + cd misc/fctl-plugin + go run ./cmd/fctl-plugin-publish --manifest ../../dist/fctl-plugin-manifest.json.tmp --artifacts ../../dist/artifacts.json --source-root ../.. --service-version "$version" --revision "$revision" > ../../dist/fctl-plugin-catalogue.json.tmp + cd ../.. + mv dist/fctl-plugin-catalogue.json.tmp dist/fctl-plugin-catalogue.json + GH_TOKEN="$GITHUB_TOKEN" gh release upload "$GITHUB_REF_NAME" dist/fctl-plugin-catalogue.json --clobber --repo "$GITHUB_REPOSITORY" + + +# Read-only check after an administrator enables public GHCR downloads. +verify-fctl-plugin-anonymous catalogue="dist/fctl-plugin-catalogue.json": + #!/usr/bin/env bash + set -euo pipefail + catalogue_path="$(realpath '{{catalogue}}')" + fixture="$(mktemp -d)" + trap 'rm -rf "$fixture"' EXIT + printf '{"auths":{}}' > "$fixture/config.json" + jq -e '.schemaVersion == 1 and (.releases | length == 6)' "$catalogue_path" > /dev/null + while IFS=$'\t' read -r registry repository digest checksum; do + reference="${registry#https://}/$repository@$digest" + oras pull --registry-config "$fixture/config.json" --output "$fixture/artifact" "$reference" + printf '%s %s\n' "$checksum" "$fixture/artifact/fctl-plugin-ledger" | sha256sum --check + rm -rf "$fixture/artifact" + done < <(jq -r '.releases[] | [.artifact.registry, .artifact.repository, .artifact.digest, .sha256] | @tsv' "$catalogue_path") + + +# Exercise the real external host without importing its core. +test-fctl-plugin-host binary: + #!/usr/bin/env bash + set -euo pipefail + fixture_tmp="$(mktemp -d /tmp/fctl-plugin.XXXXXX)" + trap 'rm -rf "$fixture_tmp"' EXIT + export TMPDIR="$fixture_tmp" + FCTL_BINARY="{{binary}}" go -C misc/fctl-plugin test -race -tags host ./cmd/fctl-plugin-ledger -run TestFctlLifecycle -count=1 -timeout 3m diff --git a/misc/fctl-plugin/.goreleaser.yml b/misc/fctl-plugin/.goreleaser.yml new file mode 100644 index 0000000000..bcdae0b2d7 --- /dev/null +++ b/misc/fctl-plugin/.goreleaser.yml @@ -0,0 +1,51 @@ +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +version: 2 +project_name: fctl-plugin-ledger +dist: ../../build/fctl-plugin + +env: + - PLUGIN_REVISION={{ envOrDefault "PLUGIN_REVISION" "1" }} + - LEDGER_SERVICE_VERSION={{ envOrDefault "LEDGER_SERVICE_VERSION" .Version }} + +builds: + - id: fctl-plugin-ledger + binary: fctl-plugin-ledger + main: ./cmd/fctl-plugin-ledger + env: + - CGO_ENABLED=0 + # Keep -ldflags build metadata: the publisher checks version/revision on + # every foreign-platform executable before uploading any of them. + ldflags: + - -s -w + - -X main.serviceVersion={{ .Env.LEDGER_SERVICE_VERSION }} + - -X main.revision={{ .Env.PLUGIN_REVISION }} + goos: + - linux + - darwin + - windows + goarch: + - amd64 + - arm64 + +archives: + - ids: + - fctl-plugin-ledger + name_template: "{{ .ProjectName }}_v{{ .Env.LEDGER_SERVICE_VERSION }}_r{{ .Env.PLUGIN_REVISION }}_{{ .Os }}-{{ .Arch }}" + formats: + - tar.gz + format_overrides: + - goos: windows + formats: + - zip + files: + - README.md + +checksum: + name_template: checksums.txt + +# The service release configuration attaches the same builds to Ledger releases. +# This configuration prepares independent plugin snapshots without publishing. +release: + disable: true +changelog: + disable: true diff --git a/misc/fctl-plugin/README.md b/misc/fctl-plugin/README.md new file mode 100644 index 0000000000..b47b36436e --- /dev/null +++ b/misc/fctl-plugin/README.md @@ -0,0 +1,204 @@ +# Ledger fctl plugin + +Profiles: **CLI** and **library**. This separate Go 1.26 module owns Ledger's +HTTP commands for fctl v4. It depends on the public `pluginsdk` module only; +it does not import Ledger server internals, Cobra, or the fctl core. + +`fctl-plugin-ledger` is a native executable. fctl starts it with the SDK +transport, renders its declarative command manifest and forms, and supplies an +HTTP client through the host broker. Authentication, endpoint selection, +request diagnostics, profiles, file input, and output styling belong to fctl. +The library also exposes `NewWithExecutor` for alternate transports; it retains +the same validation and raw request body. The external executable uses the HTTP +factory. The host acquires inputs, selects endpoints and renders results. + +The module and its Go import path are `misc/fctl-plugin` and +`github.com/formancehq/ledger/misc/fctl-plugin`. The command implementation +preserves the existing module at `060fabc5af8d`, including payload validation, +opaque page cursors and the forward `--after` alias. Routes and payload shapes +were checked against `release/v3.0` at `8e9bb3ebc13f` and `openapi.yml`. + +The plugin entry point uses the SDK-owned protocol and standard `flag` package +for its two offline metadata flags. It has no human command tree or operational +listener, so the CLI profile's Cobra and Fx composition requirements do not +apply. Owner: Ledger maintainers. Revisit this exception if the executable +acquires independent user commands or a service lifecycle. + +## Version identity + +| Value | Meaning | +| --- | --- | +| Service version | Exact version of the co-released Ledger service, such as `3.0.0-beta.5` | +| Plugin revision | Positive integer identifying a CLI-only revision for that service version | +| Protocol version | SDK runtime protocol understood by fctl and the executable | + +The SDK manifest's `version` is the service version. Plugin revision is separate +metadata; it must not be appended to the service version. `NewVersion(client, +serviceVersion)` holds the version on each instance and returns fresh manifests. +`New(client)` retains the initial `3.0.0` embedded contract. + +An independent CLI fix for Ledger `3.0.0` increments plugin revision from `1` +to `2`. A Ledger `3.0.1` release starts with its own revision `1`. Each platform +artifact is immutable and resolves to a digest; never replace an already +advertised artifact under the same service version and revision. + +## Build and test + +From the repository root, use the pinned Nix development environment: + +```sh +nix develop --command just test-fctl-plugin +nix develop --command just build-fctl-plugin 3.0.0-beta.5 1 +./build/fctl-plugin-ledger --version +./build/fctl-plugin-ledger --manifest +nix develop --command just fctl-plugin-manifest +``` + +The default local build reports service version `dev` and revision `1`. Set the +two recipe arguments to test the exact service version deployed on your target. +`--version` prints JSON with `name`, `serviceVersion`, `revision`, and +`protocolVersion`. `--manifest` prints the complete SDK command manifest and +does not start the plugin transport or access a server. The manifest recipe +exports it to `build/fctl-plugin-manifest.json`. + +Without metadata flags, the executable serves the SDK runtime protocol and is +started by fctl. It is not a replacement for the `fctl ledger ...` command tree. +Use fctl to select an authenticated Cloud stack or a local unauthenticated +endpoint, then execute Ledger commands through the selected plugin. + +All contributor tooling comes from the root Nix environment and its existing +`flake.lock`. The SDK contains the protocol implementation; plugin builds do +not require a host `protoc` installation. The root Nix environment also +provides ORAS, GitHub CLI, jq and checksum tools used for publication. + +To verify installation and execution through a trusted fctl v4 binary without +importing its core, run: + +```sh +nix develop --command just test-fctl-plugin-host /absolute/path/to/fctl +``` + +This starts controlled local HTTP fixtures and the real plugin executable. It +checks local installation, inspection, exact integer responses and requests, +partial bulk errors, version drift before mutation, and help/completion after +the API is stopped. The ordinary race suite also checks argument and payload +validation, forms, cursor handling and the SDK HTTP broker. The test recipes +allocate and clean up a short temporary root: SDK Unix sockets can exceed +macOS path limits inside the canonical validator's nested `TMPDIR`. + +## Package without publishing + +```sh +nix develop --command just package-fctl-plugin 3.0.0-beta.5 1 +``` + +GoReleaser builds six static executables: Linux, macOS, and Windows, each for +amd64 and arm64. Archives and SHA-256 checksums are written to +`build/fctl-plugin/`. Unix archives use tar.gz; Windows archives use zip. +Names include the exact Ledger service version and plugin revision. This +recipe uses snapshot mode and skips all publication. + +The release builds preserve injected version and revision in Go build +metadata. Do not add `-trimpath`: Go omits recorded linker flags with this +option, so the publisher cannot verify a foreign-platform executable's +service version and plugin revision and will reject it before upload. + +The root GoReleaser configuration also builds the separate module during each +Ledger release. It injects the actual release version and `PLUGIN_REVISION` +(default `1`) and attaches the platform archives to the Ledger GitHub release +alongside the server and ledgerctl archives. The release workflow runs the +plugin tests before building, and the normal CI test job validates the nested +module. Root lint, tidy, and agent baseline checks include this module. + +## Catalogue and OCI publication + +The SDK manifest from `--manifest` is the command contract used by the fctl +catalogue. It preserves argument rules, request-body bindings, and declarative +forms without duplicating them in a handwritten catalogue file. + +The publisher combines this SDK manifest with each raw executable's actual +SHA-256 checksum and the digest returned by ORAS. It emits the shared fctl +catalogue schema `1`: `releases` entries contain `service`, `serviceVersion`, +`revision`, `platform` (`os`, `arch`), `artifact` (`registry`, `repository`, +`digest`), `sha256`, and `manifest`. + +Each OCI image manifest has artifact type +`application/vnd.formance.fctl.plugin.v1`, an empty JSON config with media type +`application/vnd.oci.empty.v1+json`, and exactly one raw executable layer with +media type `application/vnd.formance.fctl.plugin.executable.v1`. It uses OCI +image specification `v1.1`. The executable is uploaded directly, without an +archive or container image index. Catalogue entries use immutable digests. + +Prepare a matching native manifest and six platform binaries, then exercise +ORAS locally before publishing: + +```sh +nix develop --command just build-fctl-plugin 3.0.0-beta.10 1 fctl-plugin-manifest +nix develop --command just package-fctl-plugin 3.0.0-beta.10 1 +nix develop --command just fctl-plugin-oci-layout 1 +``` + +The layout recipe writes real artifacts under `build/fctl-plugin-oci/` and +the six-platform catalogue to `build/fctl-plugin-catalogue.json`. It contacts +no registry. The catalogue still names the configured registry: a local OCI +layout is a serialization fixture, not a hosted download endpoint. + +For an explicitly authorized upload, authenticate ORAS using the publishing +account, then run: + +```sh +nix develop --command oras login ghcr.io --username PUBLISHER --password-stdin +nix develop --command just publish-fctl-plugin https://ghcr.io formancehq/fctl-plugin-ledger 1 +``` + +This recipe reads the already prepared snapshot, uploads all six raw binaries, +and replaces `build/fctl-plugin-catalogue.json` only after every upload succeeds. +It checks every executable's embedded service version, revision, product +entry point, and target platform before the first ORAS call. A stale manifest +or mixed release set produces no upload and no catalogue. +For a tagged co-release, the same helper accepts the root GoReleaser +`artifacts.json` and distribution directory through `--artifacts` and +`--source-root`. Export the native binary's manifest from that exact release. +The tag workflow invokes `just publish-fctl-plugin-release` after GoReleaser. +It executes the Linux/amd64 binary from that exact artifact list, exports its +manifest, and compares `--service-version` to `GITHUB_REF_NAME` without the `v` +prefix. The separate revision is explicitly `1` in the workflow. Registry login +uses the job token and actor with `packages: write`; no long-lived token is +required for plugin upload. Credentials use a temporary ORAS configuration +which is removed on exit. The complete `fctl-plugin-catalogue.json` is attached +to the GitHub release only after all six uploads succeed. If an upload fails, +earlier artifacts may remain in GHCR, but no new catalogue is advertised. A +release-asset upload failure leaves the complete catalogue in `dist/` for retry. +Nightly releases build archives but do not run this tagged publication step. + +## First official publication + +1. Review and merge the Draft PR into `release/v3.0` through normal repository + gates. Choose the exact service release tag and plugin revision. No tag or + release is created by development or snapshot recipes. +2. An authorized maintainer creates the Ledger tag containing this module and + workflow. The existing release job builds the service plus six plugin + executables, then publishes to `ghcr.io/formancehq/fctl-plugin-ledger` and + attaches `fctl-plugin-catalogue.json`. +3. Set the GHCR package to **Public** and allow this repository job token write + access in the package settings. Download the release catalogue and run + `nix develop --command just verify-fctl-plugin-anonymous PATH` to check all + six digests and executable checksums with empty registry credentials. +4. Open a separate reviewed promotion PR in + [formancehq/fctl-plugin-registry](https://github.com/formancehq/fctl-plugin-registry). + Add the six exact release entries to `registry.yaml`, keeping existing Auth + entries. Copy manifests, checksums, revisions and immutable digests from the + release asset; do not hand-invent them or promote the local layout fixture. +5. After promotion, verify `fctl plugins sync --service ledger` against the + matching service and inspect the resulting lock with `fctl plugins show`. + +Publishing and official catalogue promotion are separate operations. A plugin +fix for an already published service version requires a new positive revision; +retain previously advertised digests. + +GitHub creates GHCR packages privately by default. After the first upload, +set the package visibility to **Public** in its GitHub package settings. For +a private source repository, remove inherited repository permissions when +needed before changing package visibility. Keep sources private while allowing +anonymous artifact downloads. This is a one-time registry setup, independent +of source visibility; see [GitHub package permissions](https://docs.github.com/en/packages/learn-github-packages/about-permissions-for-github-packages). diff --git a/misc/fctl-plugin/cmd/fctl-plugin-catalogue/main.go b/misc/fctl-plugin/cmd/fctl-plugin-catalogue/main.go new file mode 100644 index 0000000000..20a2615ad2 --- /dev/null +++ b/misc/fctl-plugin/cmd/fctl-plugin-catalogue/main.go @@ -0,0 +1,55 @@ +// fctl-plugin-catalogue prepares a single immutable platform release entry. +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + + "github.com/formancehq/ledger/misc/fctl-plugin/internal/catalogue" +) + +func main() { + if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(args []string, output, diagnostics io.Writer) error { + flags := flag.NewFlagSet("fctl-plugin-catalogue", flag.ContinueOnError) + flags.SetOutput(diagnostics) + manifestPath := flags.String("manifest", "", "SDK manifest exported by the native plugin") + binary := flags.String("binary", "", "Raw GoReleaser platform executable to checksum") + registry := flags.String("registry", "", "Registry HTTPS origin, or loopback HTTP origin") + repository := flags.String("repository", "", "OCI repository") + digest := flags.String("digest", "", "Immutable OCI manifest sha256 digest") + goos := flags.String("os", "", "Executable operating system") + arch := flags.String("arch", "", "Executable architecture") + revision := flags.Int("revision", 1, "Positive plugin revision for this service version") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } + + return err + } + if flags.NArg() != 0 { + return fmt.Errorf("unexpected arguments: %v", flags.Args()) + } + manifest, err := catalogue.ReadManifest(*manifestPath) + if err != nil { + return err + } + release, err := catalogue.NewRelease(manifest, *binary, + catalogue.Artifact{Registry: *registry, Repository: *repository, Digest: *digest}, + catalogue.Platform{OS: *goos, Arch: *arch}, *revision) + if err != nil { + return err + } + + return json.NewEncoder(output).Encode(catalogue.Catalogue{SchemaVersion: 1, Releases: []catalogue.Release{release}}) +} diff --git a/misc/fctl-plugin/cmd/fctl-plugin-ledger/executable_test.go b/misc/fctl-plugin/cmd/fctl-plugin-ledger/executable_test.go new file mode 100644 index 0000000000..200b0d9759 --- /dev/null +++ b/misc/fctl-plugin/cmd/fctl-plugin-ledger/executable_test.go @@ -0,0 +1,98 @@ +package main + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/formancehq/fctl/pkg/pluginsdk" + "github.com/formancehq/fctl/pkg/pluginsdk/httpclient" + "github.com/formancehq/fctl/pkg/pluginsdk/transport" +) + +type authenticatedTransport struct{ base http.RoundTripper } + +func (t authenticatedTransport) RoundTrip(req *http.Request) (*http.Response, error) { + request := req.Clone(req.Context()) + request.Header.Set("Authorization", "Bearer synthetic-host-token") + + return t.base.RoundTrip(request) +} + +func TestStandaloneExecutableUsesHostHTTPBroker(t *testing.T) { + t.Parallel() + binary := filepath.Join(t.TempDir(), "fctl-plugin-ledger") + build := exec.CommandContext(t.Context(), "go", "build", "-o", binary, + "-ldflags", "-X main.serviceVersion=3.0.0-beta.5 -X main.revision=2", ".") + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("build standalone executable: %s, %v", output, err) + } + const payload = `{ "postings": [{"source":"world","destination":"users:42","asset":"USD/2","amount":123456789012345678901234567890}] }` + const result = `{ "data": { "id": 123456789012345678901234567890 } }` + const partial = `{ "data": [{"responseType":"ERROR","errorCode":"REJECTED"}], "errorCode":"BULK_FAILED", "errorMessage":"partial failure" }` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Header.Get("Authorization") != "Bearer synthetic-host-token" { + t.Error("host authentication was not applied") + } + w.Header().Set("Content-Type", "application/json") + switch req.URL.Path { + case "/v3/books/transactions": + body, err := io.ReadAll(req.Body) + if err != nil || string(body) != payload || req.Method != http.MethodPost { + t.Errorf("request changed: method=%s, body=%s, err=%v", req.Method, body, err) + } + w.WriteHeader(http.StatusCreated) + if _, err := io.WriteString(w, result); err != nil { + t.Error(err) + } + case "/v3/books/bulk": + w.WriteHeader(http.StatusBadRequest) + if _, err := io.WriteString(w, partial); err != nil { + t.Error(err) + } + default: + t.Errorf("unexpected route: %s", req.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + client := &http.Client{Transport: authenticatedTransport{base: http.DefaultTransport}} + plugin, err := transport.Open(t.Context(), binary, client, server.URL) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := plugin.Close(); err != nil { + t.Error(err) + } + }() + manifest, err := plugin.GetManifest(t.Context()) + if err != nil || manifest.Version != "3.0.0-beta.5" { + t.Fatalf("standalone manifest=%#v, err=%v", manifest, err) + } + if _, err := pluginsdk.FindCommand(manifest, []string{"ledger", "transactions", "create"}); err != nil { + t.Fatal(err) + } + response, err := plugin.Execute(t.Context(), pluginsdk.ExecuteRequest{ + CommandPath: []string{"ledger", "transactions", "create"}, + Flags: map[string]string{"ledger": "books"}, + Body: json.RawMessage(payload), Endpoint: server.URL, + }) + if err != nil || string(response.Data) != result { + t.Fatalf("standalone result=%s, err=%v", response.Data, err) + } + response, err = plugin.Execute(t.Context(), pluginsdk.ExecuteRequest{ + CommandPath: []string{"ledger", "bulk"}, Flags: map[string]string{"ledger": "books"}, + Body: json.RawMessage("[]"), Endpoint: server.URL, + }) + failure, ok := errors.AsType[*httpclient.Error](err) + if !ok || failure.StatusCode != http.StatusBadRequest || !strings.Contains(failure.Code, "BULK_FAILED") || string(response.Data) != partial { + t.Fatalf("partial standalone result=%s, err=%v", response.Data, err) + } +} diff --git a/misc/fctl-plugin/cmd/fctl-plugin-ledger/host_it_test.go b/misc/fctl-plugin/cmd/fctl-plugin-ledger/host_it_test.go new file mode 100644 index 0000000000..b8472637be --- /dev/null +++ b/misc/fctl-plugin/cmd/fctl-plugin-ledger/host_it_test.go @@ -0,0 +1,115 @@ +//go:build host + +package main + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" +) + +// TestFctlLifecycle exercises the public CLI boundary without importing fctl. +// Supply a trusted, already-built fctl v4 executable through FCTL_BINARY. +func TestFctlLifecycle(t *testing.T) { + t.Parallel() + host, err := filepath.Abs(os.Getenv("FCTL_BINARY")) + if err != nil || os.Getenv("FCTL_BINARY") == "" { + t.Fatal("FCTL_BINARY must name the fctl v4 executable") + } + binary := filepath.Join(t.TempDir(), "fctl-plugin-ledger") + build := exec.CommandContext(t.Context(), "go", "build", "-o", binary, "-ldflags", "-X main.serviceVersion=3.0.0-beta.10 -X main.revision=1", ".") + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("build plugin: %s %v", output, err) + } + var version atomic.Value + version.Store("3.0.0-beta.10") + var mutations, requests atomic.Int32 + const amount = "90071992547409930001" + const payload = `{"postings":[{"source":"world","destination":"users:42","asset":"USD/2","amount":` + amount + `}]}` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + var err error + switch r.URL.Path { + case "/gateway/ledger/_info": + err = json.NewEncoder(w).Encode(map[string]any{"version": version.Load()}) + case "/gateway/ledger/v3/": + _, err = io.WriteString(w, `{"data":[{"name":"books"}]}`) + case "/gateway/ledger/v3/books/transactions": + mutations.Add(1) + body, readErr := io.ReadAll(r.Body) + if readErr != nil || string(body) != payload || r.Method != http.MethodPost { + t.Errorf("changed mutation: %s %s %v", r.Method, body, readErr) + } + w.WriteHeader(http.StatusCreated) + _, err = io.WriteString(w, `{"data":{"id":`+amount+`}}`) + case "/gateway/ledger/v3/books/bulk": + mutations.Add(1) + w.WriteHeader(http.StatusBadRequest) + _, err = io.WriteString(w, `{"data":[{"responseType":"CREATE_TRANSACTION","data":{"id":`+amount+`}},{"responseType":"ERROR","errorCode":"INSUFFICIENT_FUNDS"}],"errorCode":"BULK_FAILED","errorMessage":"partial failure"}`) + default: + t.Errorf("unexpected route: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + if err != nil { + t.Error(err) + } + })) + defer server.Close() + args := []string{"--config-dir", t.TempDir(), "--auth-mode", "none", "--ledger-url", server.URL + "/gateway/ledger", "--no-input", "-o", "json"} + catalogue := filepath.Join(t.TempDir(), "catalogue.json") + if err := os.WriteFile(catalogue, []byte(`{"schemaVersion":1,"releases":[]}`), 0o600); err != nil { + t.Fatal(err) + } + runHost := func(extra ...string) (string, error) { + t.Helper() + command := exec.CommandContext(t.Context(), host, append(append([]string{}, args...), extra...)...) + command.Env = append(os.Environ(), "FCTL_PLUGIN_CATALOGUE="+catalogue) + output, err := command.CombinedOutput() + + return string(output), err + } + out, err := runHost("plugins", "install", "--service", "ledger", "--binary", binary) + if err != nil || !strings.Contains(out, `"source": "local"`) { + t.Fatalf("install: %s %v", out, err) + } + out, err = runHost("plugins", "show", "--service", "ledger") + if err != nil || !strings.Contains(out, "3.0.0-beta.10") { + t.Fatalf("show: %s %v", out, err) + } + out, err = runHost("ledger", "list") + if err != nil || !strings.Contains(out, "books") { + t.Fatalf("list: %s %v", out, err) + } + out, err = runHost("ledger", "transactions", "create", "--ledger", "books", "--data", payload) + if err != nil || !strings.Contains(out, amount) || mutations.Load() != 1 { + t.Fatalf("create: %s %v mutations=%d", out, err, mutations.Load()) + } + out, err = runHost("ledger", "bulk", "--ledger", "books", "--data", "[]") + if err == nil || !strings.Contains(out, amount) || !strings.Contains(out, "BULK_FAILED") || mutations.Load() != 2 { + t.Fatalf("partial result: %s %v mutations=%d", out, err, mutations.Load()) + } + version.Store("3.0.0-beta.11") + out, err = runHost("ledger", "transactions", "create", "--ledger", "books", "--data", payload) + if err == nil || !strings.Contains(out, "exact service version") || mutations.Load() != 2 { + t.Fatalf("version drift reached mutation: %s %v mutations=%d", out, err, mutations.Load()) + } + server.Close() + before := requests.Load() + for _, extra := range [][]string{{"ledger", "--help"}, {"ledger", "transactions", "create", "--help"}, {"__complete", "ledger", "transactions", ""}} { + out, err = runHost(extra...) + if err != nil || out == "" || requests.Load() != before { + t.Fatalf("offline metadata %v: %s %v", extra, out, err) + } + if extra[0] == "__complete" && !strings.Contains(out, "create") { + t.Fatalf("missing completion: %s", out) + } + } +} diff --git a/misc/fctl-plugin/cmd/fctl-plugin-ledger/main.go b/misc/fctl-plugin/cmd/fctl-plugin-ledger/main.go new file mode 100644 index 0000000000..889645ee6a --- /dev/null +++ b/misc/fctl-plugin/cmd/fctl-plugin-ledger/main.go @@ -0,0 +1,86 @@ +// fctl-plugin-ledger serves Ledger commands through the public fctl protocol. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/http" + "os" + "strconv" + + "github.com/formancehq/fctl/pkg/pluginsdk" + "github.com/formancehq/fctl/pkg/pluginsdk/transport" + + ledger "github.com/formancehq/ledger/misc/fctl-plugin" +) + +// GoReleaser supplies both values for each immutable product release artifact. +var ( + serviceVersion = "dev" + revision = "1" +) + +type versionMetadata struct { + Name string `json:"name"` + ServiceVersion string `json:"serviceVersion"` + Revision int `json:"revision"` + ProtocolVersion int `json:"protocolVersion"` +} + +func main() { + if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(args []string, output, diagnostics io.Writer) error { + flags := flag.NewFlagSet("fctl-plugin-ledger", flag.ContinueOnError) + flags.SetOutput(diagnostics) + manifest := flags.Bool("manifest", false, "Print the SDK command manifest as JSON without starting the plugin") + version := flags.Bool("version", false, "Print the service version and plugin revision as JSON") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } + + return err + } + if flags.NArg() != 0 { + return fmt.Errorf("unexpected arguments: %v", flags.Args()) + } + if *manifest && *version { + return errors.New("use either --manifest or --version") + } + pluginRevision, err := strconv.Atoi(revision) + if err != nil || pluginRevision < 1 { + return errors.New("plugin revision must be a positive integer") + } + if serviceVersion == "" { + return errors.New("ledger service version must not be empty") + } + factory := func(client *http.Client) pluginsdk.Plugin { + return ledger.NewVersion(client, serviceVersion) + } + if *manifest { + metadata, err := factory(nil).GetManifest(context.Background()) + if err != nil { + return err + } + + return json.NewEncoder(output).Encode(metadata) + } + if *version { + return json.NewEncoder(output).Encode(versionMetadata{ + Name: "ledger", ServiceVersion: serviceVersion, + Revision: pluginRevision, ProtocolVersion: pluginsdk.ProtocolVersion, + }) + } + transport.Serve(factory) + + return nil +} diff --git a/misc/fctl-plugin/cmd/fctl-plugin-ledger/main_test.go b/misc/fctl-plugin/cmd/fctl-plugin-ledger/main_test.go new file mode 100644 index 0000000000..493c31e5bf --- /dev/null +++ b/misc/fctl-plugin/cmd/fctl-plugin-ledger/main_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "bytes" + "encoding/json" + "io" + "testing" + + "github.com/formancehq/fctl/pkg/pluginsdk" +) + +func TestMetadataFlags(t *testing.T) { + t.Parallel() + for _, flag := range []string{"--manifest", "--version"} { + t.Run(flag, func(t *testing.T) { + t.Parallel() + var output bytes.Buffer + if err := run([]string{flag}, &output, io.Discard); err != nil { + t.Fatal(err) + } + if flag == "--manifest" { + var manifest pluginsdk.Manifest + if err := json.Unmarshal(output.Bytes(), &manifest); err != nil { + t.Fatal(err) + } + if manifest.Name != "ledger" || manifest.Version != serviceVersion || manifest.ProtocolVersion != pluginsdk.ProtocolVersion { + t.Fatalf("unexpected manifest: %#v", manifest) + } + if _, err := pluginsdk.FindCommand(manifest, []string{"ledger", "transactions", "create"}); err != nil { + t.Fatal(err) + } + + return + } + var version versionMetadata + if err := json.Unmarshal(output.Bytes(), &version); err != nil { + t.Fatal(err) + } + if version.Name != "ledger" || version.ServiceVersion != serviceVersion || version.Revision != 1 || version.ProtocolVersion != pluginsdk.ProtocolVersion { + t.Fatalf("unexpected version metadata: %#v", version) + } + }) + } +} + +func TestInvalidMetadataArguments(t *testing.T) { + t.Parallel() + for _, args := range [][]string{{"--manifest", "--version"}, {"--manifest", "extra"}, {"--unknown"}} { + if err := run(args, io.Discard, io.Discard); err == nil { + t.Fatalf("accepted invalid arguments: %v", args) + } + } + if err := run([]string{"--help"}, io.Discard, io.Discard); err != nil { + t.Fatal(err) + } +} diff --git a/misc/fctl-plugin/cmd/fctl-plugin-publish/main.go b/misc/fctl-plugin/cmd/fctl-plugin-publish/main.go new file mode 100644 index 0000000000..cf43725ca9 --- /dev/null +++ b/misc/fctl-plugin/cmd/fctl-plugin-publish/main.go @@ -0,0 +1,46 @@ +// fctl-plugin-publish uploads GoReleaser binaries and emits catalogue schema 1. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + + "github.com/formancehq/ledger/misc/fctl-plugin/internal/catalogue" +) + +func main() { + if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(args []string, output, diagnostics io.Writer) error { + flags := flag.NewFlagSet("fctl-plugin-publish", flag.ContinueOnError) + flags.SetOutput(diagnostics) + var options catalogue.PublishOptions + flags.StringVar(&options.ServiceVersion, "service-version", "", "Exact co-released Ledger service version (without v prefix)") + flags.StringVar(&options.ManifestPath, "manifest", "", "SDK manifest exported by the native plugin") + flags.StringVar(&options.ArtifactsPath, "artifacts", "", "GoReleaser artifacts.json") + flags.StringVar(&options.SourceRoot, "source-root", "", "Root containing raw GoReleaser binaries") + flags.StringVar(&options.Registry, "registry", "https://ghcr.io", "Public OCI registry HTTPS origin") + flags.StringVar(&options.Repository, "repository", "formancehq/fctl-plugin-ledger", "Public OCI repository") + flags.StringVar(&options.Layout, "layout", "", "Write a local OCI image layout instead of pushing to the registry") + flags.IntVar(&options.Revision, "revision", 1, "Positive plugin revision for the exact Ledger version") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } + + return err + } + if flags.NArg() != 0 { + return fmt.Errorf("unexpected arguments: %v", flags.Args()) + } + + return catalogue.Publish(context.Background(), options, output, diagnostics) +} diff --git a/misc/fctl-plugin/cmd/fctl-plugin-publish/main_test.go b/misc/fctl-plugin/cmd/fctl-plugin-publish/main_test.go new file mode 100644 index 0000000000..72237d1974 --- /dev/null +++ b/misc/fctl-plugin/cmd/fctl-plugin-publish/main_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + ledger "github.com/formancehq/ledger/misc/fctl-plugin" +) + +func TestRunHelpDocumentsPublisherFlags(t *testing.T) { + t.Parallel() + var output, diagnostics bytes.Buffer + if err := run([]string{"--help"}, &output, &diagnostics); err != nil { + t.Fatal(err) + } + for _, flag := range []string{"service-version", "manifest", "artifacts", "source-root", "registry", "repository", "layout", "revision"} { + if !strings.Contains(diagnostics.String(), "-"+flag) { + t.Errorf("help omits --%s: %s", flag, &diagnostics) + } + } + if !strings.Contains(diagnostics.String(), "formancehq/fctl-plugin-ledger") || output.Len() != 0 { + t.Fatalf("help must document Ledger repository on diagnostics only: output=%s diagnostics=%s", &output, &diagnostics) + } +} + +func TestRunRejectsInvalidArgumentsWithoutCatalogue(t *testing.T) { + t.Parallel() + for _, args := range [][]string{{"--unknown"}, {"--revision", "invalid"}, {"unexpected"}} { + t.Run(strings.Join(args, " "), func(t *testing.T) { + var output, diagnostics bytes.Buffer + if err := run(args, &output, &diagnostics); err == nil { + t.Fatal("invalid arguments accepted") + } + if output.Len() != 0 { + t.Fatalf("invalid arguments advertised catalogue: %s", &output) + } + }) + } +} + +func TestRunRequiresServiceVersionAndPassesExactVersionToPublisher(t *testing.T) { + t.Parallel() + manifest, err := ledger.NewVersion(nil, "3.0.0-beta.10").GetManifest(t.Context()) + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + directory := t.TempDir() + manifestPath := filepath.Join(directory, "manifest.json") + if err := os.WriteFile(manifestPath, data, 0o600); err != nil { + t.Fatal(err) + } + artifactsPath := filepath.Join(directory, "artifacts.json") + if err := os.WriteFile(artifactsPath, []byte("[]"), 0o600); err != nil { + t.Fatal(err) + } + for _, version := range []string{"", "3.0.1", "v3.0.0-beta.10", manifest.Version} { + t.Run("version="+version, func(t *testing.T) { + assertPublisherVersion(t, directory, manifestPath, artifactsPath, version, manifest.Version) + }) + } +} + +func assertPublisherVersion(t *testing.T, directory, manifestPath, artifactsPath, version, manifestVersion string) { + t.Helper() + layout := filepath.Join(t.TempDir(), "must-not-publish") + args := []string{"--manifest", manifestPath, "--artifacts", artifactsPath, "--source-root", directory, + "--registry", "https://ghcr.io", "--repository", "formancehq/fctl-plugin-ledger", "--revision", "2", "--layout", layout} + if version != "" { + args = append(args, "--service-version", version) + } + var output, diagnostics bytes.Buffer + err := run(args, &output, &diagnostics) + want := "service version" + if version == manifestVersion { + // Matching version must reach artifact preflight, proving flag wiring. + want = "expected six" + } + if err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("expected %q error, got %v", want, err) + } + if output.Len() != 0 { + t.Fatalf("failed preflight advertised catalogue: %s", &output) + } + if _, err := os.Stat(layout); !os.IsNotExist(err) { + t.Fatalf("failed preflight reached publication: %v", err) + } +} diff --git a/misc/fctl-plugin/contract_test.go b/misc/fctl-plugin/contract_test.go new file mode 100644 index 0000000000..4b37c1aa46 --- /dev/null +++ b/misc/fctl-plugin/contract_test.go @@ -0,0 +1,183 @@ +package ledger + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "reflect" + "strings" + "testing" + + "github.com/formancehq/fctl/pkg/pluginsdk" +) + +func TestManifestContract(t *testing.T) { + t.Parallel() + p := New(nil) + manifest, err := p.GetManifest(t.Context()) + if err != nil { + t.Fatal(err) + } + if manifest.Name != "ledger" || manifest.Service != "ledger" || manifest.Version == "" || manifest.ProtocolVersion != pluginsdk.ProtocolVersion { + t.Fatalf("manifest identity: %#v", manifest) + } + data, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + var decoded pluginsdk.Manifest + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(manifest, decoded) { + t.Fatal("manifest does not survive protocol serialization") + } + create, err := pluginsdk.FindCommand(manifest, []string{"ledger", "transactions", "create"}) + if err != nil { + t.Fatal(err) + } + checkBodyFlag(t, create) + deleteCommand, err := pluginsdk.FindCommand(manifest, []string{"ledger", "accounts", "metadata", "delete"}) + if err != nil || !deleteCommand.Confirm || deleteCommand.Args.Min != 2 || deleteCommand.Args.Max != 2 { + t.Fatalf("delete contract: %#v, %v", deleteCommand, err) + } + manifest.Root.Flags[0].Name = "corrupted" + fresh, err := p.GetManifest(t.Context()) + if err != nil || fresh.Root.Flags[0].Name != "ledger" { + t.Fatal("caller mutated subsequent manifests") + } +} + +func checkBodyFlag(t *testing.T, spec pluginsdk.CommandSpec) { + t.Helper() + for _, flag := range spec.Flags { + if flag.Name == "data" { + if !flag.Required || !flag.Body || flag.Type != "string" { + t.Fatalf("data flag: %#v", flag) + } + + return + } + } + t.Fatal("transaction creation missing data flag") +} + +type transportFunc func(*http.Request) (*http.Response, error) + +func (f transportFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } + +func TestChangedFlagsAndRequestIsolation(t *testing.T) { + t.Parallel() + p := New(&http.Client{Transport: transportFunc(func(req *http.Request) (*http.Response, error) { + query := req.URL.Query() + if query.Has("reverse") || query.Get("pageSize") != "100" { + t.Errorf("unchanged flag or default query: %v", query) + } + + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(envelope))}, nil + })}) + req := pluginsdk.ExecuteRequest{CommandPath: []string{"ledger", "transactions", "list"}, Flags: map[string]string{"ledger": "books", "reverse": "true"}, Endpoint: "https://ledger.example/prefix"} + result, err := p.Execute(t.Context(), req) + if err != nil || string(result.Data) != envelope { + t.Fatalf("result=%s err=%v", result.Data, err) + } + if _, exists := req.Flags["page-size"]; exists { + t.Fatal("Execute mutated caller flags") + } +} + +func TestDirectSDKRejectsMissingBody(t *testing.T) { + t.Parallel() + p := New(nil) + req := pluginsdk.ExecuteRequest{CommandPath: []string{"ledger", "transactions", "create"}, Flags: map[string]string{"ledger": "books", "data": "@unread.json"}, Endpoint: "https://ledger.example"} + if _, err := p.Execute(t.Context(), req); err == nil { + t.Fatal("plugin attempted to execute unread body") + } +} + +func TestDirectSDKAcceptsBodyWithoutDataFlag(t *testing.T) { + t.Parallel() + body := `{"postings":[{"source":"world","destination":"users:42","asset":"USD/2","amount":123456789012345678901234567890}]}` + p := New(&http.Client{Transport: transportFunc(func(req *http.Request) (*http.Response, error) { + data, err := io.ReadAll(req.Body) + if err != nil || string(data) != body { + t.Errorf("request body=%s err=%v", data, err) + } + + return &http.Response{StatusCode: http.StatusCreated, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(envelope))}, nil + })}) + req := pluginsdk.ExecuteRequest{CommandPath: []string{"ledger", "transactions", "create"}, Flags: map[string]string{"ledger": "books"}, Body: json.RawMessage(body), Endpoint: "https://ledger.example"} + response, err := p.Execute(t.Context(), req) + if err != nil || string(response.Data) != envelope { + t.Fatalf("direct SDK body: response=%s err=%v", response.Data, err) + } +} + +func TestBulkPartialHTTPResponsePreserved(t *testing.T) { + t.Parallel() + body := `{"data":[{"responseType":"CREATE_TRANSACTION","data":{"amount":123456789012345678901234567890}},{"responseType":"ERROR","errorCode":"INSUFFICIENT_FUNDS"}],"errorCode":"BULK_FAILED","errorMessage":"partial failure"}` + out, reqs, err := execute(t, []string{"--ledger", "books", "bulk", "--data", "[]", "--idempotency-key", "batch-42"}, "", body, 400) + if err == nil || len(reqs) != 1 || out != body { + t.Fatalf("partial results lost: out=%s calls=%d err=%v", out, len(reqs), err) + } +} + +func TestHTTPFailureResponseParity(t *testing.T) { + t.Parallel() + body := `{"data":[{"responseType":"ERROR","errorCode":"REJECTED"}],"errorCode":"REJECTED","errorMessage":"request refused"}` + cases := []struct { + name string + args []string + want string + }{ + {"nonbulk response empty", []string{"--ledger", "books", "transactions", "create", "--data", "{}"}, ""}, + {"bulk partial response retained", []string{"--ledger", "books", "bulk", "--data", "[]"}, body}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + out, reqs, err := execute(t, tc.args, "", body, 400) + if err == nil || out != tc.want || len(reqs) != 1 { + t.Fatalf("error response parity: Data=%q want=%q calls=%d err=%v", out, tc.want, len(reqs), err) + } + }) + } +} + +type closeErrorReader struct { + io.Reader + + err error +} + +func (r closeErrorReader) Close() error { return r.err } + +func TestSuccessfulJSONWithCloseError(t *testing.T) { + t.Parallel() + closeErr := errors.New("response close failed") + body := `{"data":[{"responseType":"CREATE_TRANSACTION","data":{"amount":123456789012345678901234567890}}]}` + cases := []struct { + name string + path []string + body string + want string + }{ + {"nonbulk clears data", []string{"ledger", "transactions", "create"}, `{}`, ""}, + {"bulk retains data", []string{"ledger", "bulk"}, `[]`, body}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + p := New(&http.Client{Transport: transportFunc(func(*http.Request) (*http.Response, error) { + calls++ + + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: closeErrorReader{Reader: strings.NewReader(body), err: closeErr}}, nil + })}) + req := pluginsdk.ExecuteRequest{CommandPath: tc.path, Flags: map[string]string{"ledger": "books"}, Body: json.RawMessage(tc.body), Endpoint: "https://ledger.example"} + response, err := p.Execute(t.Context(), req) + if !errors.Is(err, closeErr) || string(response.Data) != tc.want || calls != 1 { + t.Fatalf("close error: Data=%q want=%q calls=%d err=%v", response.Data, tc.want, calls, err) + } + }) + } +} diff --git a/misc/fctl-plugin/executor_test.go b/misc/fctl-plugin/executor_test.go new file mode 100644 index 0000000000..c1fa8fba63 --- /dev/null +++ b/misc/fctl-plugin/executor_test.go @@ -0,0 +1,252 @@ +package ledger + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/formancehq/fctl/pkg/pluginsdk" +) + +func TestExecutorPreservesPreparedJSONAndPartialResults(t *testing.T) { + t.Parallel() + const body = `{"postings":[{"source":"world","destination":"users:42","asset":"USD/2","amount":9007199254740993}]}` + const partial = `{"data":{"id":9007199254740993},"errorCode":"PARTIAL_FAILURE"}` + failure := errors.New("accepted write, response delivery failed") + calls := 0 + p := NewWithExecutor("3.0.0-beta.10", func(ctx context.Context, req pluginsdk.ExecuteRequest) (pluginsdk.ExecuteResponse, error) { + calls++ + if ctx != t.Context() || req.Endpoint != "grpc://ledger.example:8080" || string(req.Body) != body { + t.Fatalf("executor changed context, endpoint or exact JSON: %#v", req) + } + if req.Flags["data"] != "@already-read.json" || !req.ChangedFlags["data"] || req.Flags["idempotency-key"] != "payment-42" || req.Flags["consistency"] != "" { + t.Fatalf("executor request was not normalized or lost flags: %#v", req) + } + if req.Context["profile"] != "local" { + t.Fatalf("host context lost: %#v", req.Context) + } + + return pluginsdk.ExecuteResponse{Data: json.RawMessage(partial)}, failure + }) + req := pluginsdk.ExecuteRequest{ + CommandPath: []string{"ledger", "transactions", "create"}, + Flags: map[string]string{"ledger": "books", "data": "@already-read.json", "idempotency-key": "payment-42"}, + ChangedFlags: map[string]bool{"data": true}, + Body: json.RawMessage(body), Endpoint: "grpc://ledger.example:8080", Context: map[string]string{"profile": "local"}, + } + response, err := p.Execute(t.Context(), req) + if !errors.Is(err, failure) || string(response.Data) != partial || calls != 1 { + t.Fatalf("partial result, exact number or single attempt lost: response=%s calls=%d err=%v", response.Data, calls, err) + } + if _, changed := req.Flags["consistency"]; changed { + t.Fatal("normalization changed the caller's flags") + } +} + +func TestExecutorReceivesDefaultsAndExplicitFalse(t *testing.T) { + t.Parallel() + cases := []struct { + name string + req pluginsdk.ExecuteRequest + }{ + {"read defaults", pluginsdk.ExecuteRequest{CommandPath: []string{"ledger", "transactions", "list"}, Flags: map[string]string{"ledger": "books", "reverse": "false"}, ChangedFlags: map[string]bool{"reverse": true}}}, + {"create default body", pluginsdk.ExecuteRequest{CommandPath: []string{"ledger", "create"}, Args: []string{"books"}}}, + {"read ignores unused body", pluginsdk.ExecuteRequest{CommandPath: []string{"ledger", "list"}, Body: json.RawMessage(`{"unused":true}`)}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + calls := 0 + p := NewWithExecutor("", func(_ context.Context, req pluginsdk.ExecuteRequest) (pluginsdk.ExecuteResponse, error) { + calls++ + switch tc.name { + case "read defaults": + if req.Flags["page-size"] != "100" || req.Flags["reverse"] != "false" || !req.ChangedFlags["reverse"] || req.Body != nil { + t.Fatalf("default or explicit false lost: %#v", req) + } + case "create default body": + if string(req.Body) != "{}" || req.Flags["data"] != "{}" || req.ChangedFlags["data"] { + t.Fatalf("default body not prepared: %#v", req) + } + case "read ignores unused body": + if req.Body != nil { + t.Fatalf("read received a body: %s", req.Body) + } + } + + return pluginsdk.ExecuteResponse{Data: json.RawMessage(`{"data":[]}`)}, nil + }) + if _, err := p.Execute(t.Context(), tc.req); err != nil || calls != 1 { + t.Fatalf("execute: calls=%d err=%v", calls, err) + } + }) + } +} + +func TestExecutorRejectsInvalidCommandsBeforeDispatch(t *testing.T) { + t.Parallel() + cases := []struct { + name string + tokens []string + body string + want string + }{ + {"unknown command", []string{"unknown"}, "", "not executable"}, + {"unknown flag", []string{"list", "--unknown", "value"}, "", "unknown plugin flag"}, + {"missing ledger", []string{"accounts", "list"}, "", "select a ledger"}, + {"conflicting ledger", []string{"--ledger", "books", "show", "other"}, "", "conflicts with --ledger"}, + {"missing argument", []string{"--ledger", "books", "accounts", "show"}, "", "command expects"}, + {"empty identifier", []string{"--ledger", "books", "accounts", "show", ""}, "", "identifiers must not be empty"}, + {"invalid transaction ID", []string{"--ledger", "books", "transactions", "show", "-1"}, "", "unsigned 64-bit integer"}, + {"overflow transaction ID", []string{"--ledger", "books", "transactions", "show", "18446744073709551616"}, "", "unsigned 64-bit integer"}, + {"malformed JSON", []string{"--ledger", "books", "transactions", "create"}, "{", "valid JSON"}, + {"multiple JSON values", []string{"--ledger", "books", "transactions", "create"}, "{} {}", "valid JSON"}, + {"oversized body", []string{"--ledger", "books", "transactions", "create"}, `"` + strings.Repeat("x", 4<<20) + `"`, "within 4 MiB"}, + {"unread file", []string{"--ledger", "books", "transactions", "create", "--data", "@/nonexistent/request.json"}, "", "already-read JSON request body"}, + {"unread stdin", []string{"--ledger", "books", "transactions", "create", "--data", "-"}, "", "already-read JSON request body"}, + {"missing required body", []string{"--ledger", "books", "transactions", "create"}, "", "--data is required"}, + {"empty explicit optional body", []string{"--ledger", "books", "transactions", "revert", "42", "--data", ""}, "", "valid JSON"}, + {"unconfirmed delete", []string{"delete", "books"}, "", "requires --confirm"}, + {"invalid consistency", []string{"--ledger", "books", "stats", "--consistency", "eventual"}, "", "consistency must be"}, + {"invalid idempotency header", []string{"create", "books", "--idempotency-key", "a\nb"}, "", "without newlines"}, + {"invalid query type", []string{"--ledger", "books", "accounts", "list", "--page-size", "-1"}, "", "invalid --page-size"}, + {"invalid after ID", []string{"--ledger", "books", "transactions", "list", "--after", "-1"}, "", "after must be"}, + {"invalid date", []string{"--ledger", "books", "transactions", "list", "--start-date", "yesterday"}, "", "startDate must be"}, + {"invalid inspection mode", []string{"--ledger", "books", "indexes", "inspect", "index", "--mode", "bad"}, "", "mode must be"}, + {"invalid inspection page size", []string{"--ledger", "books", "indexes", "inspect", "index", "--page-size", "0"}, "", "index page-size must be"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + calls := 0 + p := NewWithExecutor("", func(context.Context, pluginsdk.ExecuteRequest) (pluginsdk.ExecuteResponse, error) { + calls++ + + return pluginsdk.ExecuteResponse{}, nil + }) + manifest, err := p.GetManifest(t.Context()) + if err != nil { + t.Fatal(err) + } + _, err = p.Execute(t.Context(), testRequest(manifest, tc.tokens, tc.body)) + if err == nil || !strings.Contains(err.Error(), tc.want) || calls != 0 { + t.Fatalf("want %q before dispatch, calls=%d err=%v", tc.want, calls, err) + } + }) + } +} + +func TestExecutorManifestIndependentOfTransport(t *testing.T) { + t.Parallel() + p := NewWithExecutor("3.0.1", nil) + manifest, err := p.GetManifest(t.Context()) + if err != nil { + t.Fatal(err) + } + httpManifest, err := NewVersion(nil, "3.0.1").GetManifest(t.Context()) + if err != nil || !reflect.DeepEqual(manifest, httpManifest) { + t.Fatalf("transport changed command or form definitions: err=%v", err) + } + manifest.Root.Flags[0].Name = "corrupted" + fresh, err := p.GetManifest(t.Context()) + if err != nil || fresh.Root.Flags[0].Name != "ledger" { + t.Fatal("caller changed a subsequent manifest") + } + if _, err := p.Execute(t.Context(), pluginsdk.ExecuteRequest{CommandPath: []string{"ledger", "list"}}); err == nil || !strings.Contains(err.Error(), "requires an executor") { + t.Fatalf("nil executor error = %v", err) + } + defaults, err := NewWithExecutor("", nil).GetManifest(t.Context()) + if err != nil || defaults.Version != defaultServiceVersion { + t.Fatalf("default service version = %q, err=%v", defaults.Version, err) + } +} + +func TestExecutorCancellation(t *testing.T) { + t.Parallel() + t.Run("before dispatch", func(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + calls := 0 + p := NewWithExecutor("", func(context.Context, pluginsdk.ExecuteRequest) (pluginsdk.ExecuteResponse, error) { + calls++ + + return pluginsdk.ExecuteResponse{}, nil + }) + _, err := p.Execute(ctx, pluginsdk.ExecuteRequest{CommandPath: []string{"ledger", "list"}}) + if !errors.Is(err, context.Canceled) || calls != 0 { + t.Fatalf("canceled request dispatched: calls=%d err=%v", calls, err) + } + }) + t.Run("in flight preserves partial data", func(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + started := make(chan struct{}) + type outcome struct { + response pluginsdk.ExecuteResponse + err error + } + done := make(chan outcome, 1) + const partial = `{"data":{"id":9007199254740993}}` + p := NewWithExecutor("", func(ctx context.Context, _ pluginsdk.ExecuteRequest) (pluginsdk.ExecuteResponse, error) { + close(started) + <-ctx.Done() + + return pluginsdk.ExecuteResponse{Data: json.RawMessage(partial)}, ctx.Err() + }) + go func() { + response, err := p.Execute(ctx, pluginsdk.ExecuteRequest{CommandPath: []string{"ledger", "list"}}) + done <- outcome{response: response, err: err} + }() + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("executor did not start") + } + cancel() + select { + case result := <-done: + if !errors.Is(result.err, context.Canceled) || string(result.response.Data) != partial { + t.Fatalf("in-flight cancellation or partial data lost: data=%s err=%v", result.response.Data, result.err) + } + case <-time.After(5 * time.Second): + t.Fatal("executor did not stop after cancellation") + } + }) +} + +func TestHTTPExecutorRetainsEndpointValidationOrder(t *testing.T) { + t.Parallel() + for _, canceled := range []bool{false, true} { + name := "active context" + if canceled { + name = "canceled context" + } + t.Run(name, func(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + if canceled { + cancel() + } + req := pluginsdk.ExecuteRequest{ + CommandPath: []string{"ledger", "transactions", "create"}, + Flags: map[string]string{"ledger": "books", "data": "@unread.json"}, + Endpoint: "invalid", + } + // Existing HTTP execution validates the endpoint before preparing an + // unread body or observing cancellation in the HTTP request. + for _, p := range []pluginsdk.Plugin{New(nil), NewVersion(nil, "3.0.1")} { + _, err := p.Execute(ctx, req) + if err == nil || !strings.Contains(err.Error(), "endpoint must be an absolute HTTP(S) URL") { + t.Fatalf("HTTP validation order changed: %v", err) + } + } + }) + } +} diff --git a/misc/fctl-plugin/go.mod b/misc/fctl-plugin/go.mod new file mode 100644 index 0000000000..ecea573efe --- /dev/null +++ b/misc/fctl-plugin/go.mod @@ -0,0 +1,22 @@ +module github.com/formancehq/ledger/misc/fctl-plugin + +go 1.26.0 + +require github.com/formancehq/fctl/pkg/pluginsdk v0.0.0-20261009101655-a5cfb893d64d + +require ( + github.com/fatih/color v1.13.0 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect + github.com/hashicorp/go-plugin v1.8.0 // indirect + github.com/hashicorp/yamux v0.1.2 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/oklog/run v1.1.0 // indirect + golang.org/x/net v0.57.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect + google.golang.org/grpc v1.84.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) diff --git a/misc/fctl-plugin/go.sum b/misc/fctl-plugin/go.sum new file mode 100644 index 0000000000..1fdd8b9ffd --- /dev/null +++ b/misc/fctl-plugin/go.sum @@ -0,0 +1,58 @@ +github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= +github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/formancehq/fctl/pkg/pluginsdk v0.0.0-20261009101655-a5cfb893d64d h1:mohqPs8upfatkGPRVHTaqYGUqQdioeN7KjF2O6qTBs4= +github.com/formancehq/fctl/pkg/pluginsdk v0.0.0-20261009101655-a5cfb893d64d/go.mod h1:u465cjRJPe7PdlPwMk4b7QiyMqmu1rdJARu8LobuQgA= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-plugin v1.8.0 h1:ie8S6RRY8RvB2usYZv+AAZ/wBvx2AU5p5QeP5j/FORs= +github.com/hashicorp/go-plugin v1.8.0/go.mod h1:BExt6KEaIYx804z8k4gRzRLEvxKVb+kn0NMcihqOqb8= +github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8= +github.com/hashicorp/yamux v0.1.2/go.mod h1:C+zze2n6e/7wshOZep2A70/aQU6QBRWJO/G6FT1wIns= +github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5XumQh94= +github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= +github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 h1:qEHAMpSaUhtD0p3NbEEI83HwNGFxEwaSJ1G9PLnCBZE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= +google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/misc/fctl-plugin/interaction.go b/misc/fctl-plugin/interaction.go new file mode 100644 index 0000000000..e8c6b03968 --- /dev/null +++ b/misc/fctl-plugin/interaction.go @@ -0,0 +1,103 @@ +package ledger + +import ( + "net/http" + "strings" + + "github.com/formancehq/fctl/pkg/pluginsdk" +) + +func ledgerInput(op operation) pluginsdk.InputSpec { + input := pluginsdk.InputSpec{Title: "Ledger", Kind: "select", Flag: "ledger", ValueType: "string", Required: true, + Source: &pluginsdk.ChoiceSource{CommandPath: []string{"ledger", "list"}, ValueField: "name", LabelFields: []string{"name"}, EmptyMessage: "No ledgers are available; create a ledger first"}} + if op.ledgerArg { + input.AlternativeArgument = new(0) + if op.method == http.MethodPost { + input.Title, input.Kind, input.Source = "Ledger name", "input", nil + } + } + + return input +} + +func operationInputs(path []string, op operation) []pluginsdk.InputSpec { + inputs := argumentInputs(path, op) + switch strings.Join(path, " ") { + case "ledger create": + inputs = append(inputs, ledgerCreateInputs()...) + case "ledger transactions create": + inputs = append(inputs, + pluginsdk.InputSpec{Title: "Numscript", Description: "Adapt the example's amount, asset and destination before submitting. Use --data for postings or scriptReference.", Kind: "text", BodyPointer: "/script/plain", ValueType: "string", Required: true, + Default: "send [USD/2 100] (\n source = @world\n destination = @users:001\n)"}, + pluginsdk.InputSpec{Title: "Reference", Description: "Optional unique transaction reference", Kind: "input", BodyPointer: "/reference", ValueType: "string"}, + pluginsdk.InputSpec{Title: "Metadata", Description: `Optional JSON object, e.g. {"order_id":"order-42"}; keep integer values exact`, Kind: "text", BodyPointer: "/metadata", ValueType: "json"}, + pluginsdk.InputSpec{Title: "Force execution", Description: "Bypass balance checks; this can overdraw accounts", Kind: "confirm", BodyPointer: "/force", ValueType: "bool", Default: "false"}, + ) + case "ledger indexes create": + // Index list IDs are protobuf objects, not canonical strings. Creating + // an index must also permit IDs that do not exist yet. + inputs = append(inputs, pluginsdk.InputSpec{Title: "Canonical index ID", Description: indexIDHelp, Kind: "input", BodyPointer: "/id", ValueType: "string", Required: true, Default: "log_builtin:LOG_BUILTIN_INDEX_DATE"}) + case "ledger metadata set", "ledger accounts metadata set", "ledger transactions metadata set": + inputs = append(inputs, pluginsdk.InputSpec{Title: "Metadata JSON", Description: `JSON object to merge, e.g. {"purpose":"automation"}; existing keys are preserved`, Kind: "text", Flag: "data", ValueType: "json", Required: true, Default: `{"purpose":"automation"}`}) + case "ledger bulk": + inputs = append(inputs, pluginsdk.InputSpec{Title: "Bulk operations JSON", Description: "JSON array of v3 operations; each operation may provide its own ik", Kind: "text", Flag: "data", ValueType: "json", Required: true, + Default: `[{"action":"CREATE_TRANSACTION","ik":"payment-42","data":{"postings":[{"source":"world","destination":"users:001","asset":"USD/2","amount":100}]}}]`}) + } + + return inputs +} + +func ledgerCreateInputs() []pluginsdk.InputSpec { + return []pluginsdk.InputSpec{ + {Title: "Account type enforcement", Description: "STRICT rejects transactions that violate account types; AUDIT records violations and permits the transaction", Kind: "select", BodyPointer: "/defaultEnforcementMode", ValueType: "string", Default: "STRICT", + Options: []pluginsdk.InputOption{{Label: "Strict (server default)", Value: "STRICT"}, {Label: "Audit", Value: "AUDIT"}}}, + {Title: "Ledger metadata", Description: `Optional JSON object with string, integer or boolean values. Creation metadata follows this co-released Ledger API contract.`, Kind: "text", BodyPointer: "/metadata", ValueType: "json"}, + {Title: "Initial metadata schema", Description: `Optional JSON array of field declarations, e.g. [{"targetType":"account","key":"color","type":"string"}]; targets: account, transaction or ledger`, Kind: "text", BodyPointer: "/initialSchema", ValueType: "json"}, + {Title: "Account types", Description: `Optional JSON object of account type models, e.g. {"user-checking":{"name":"user-checking","pattern":"users:{id}:checking","persistence":"EPHEMERAL","segmentTypes":{"id":{"type":"uint64"}}}}`, Kind: "text", BodyPointer: "/accountTypes", ValueType: "json"}, + } +} + +func argumentInputs(path []string, op operation) []pluginsdk.InputSpec { + if op.ledgerArg || op.args == 0 { + return nil + } + if path[1] == "accounts" || path[1] == "transactions" { + input := resourceInput(path[1]) + inputs := []pluginsdk.InputSpec{input} + if path[len(path)-2] == "metadata" && path[len(path)-1] == "delete" { + inputs = append(inputs, metadataKeyInput(1)) + } + + return inputs + } + if path[1] == "metadata" { + return []pluginsdk.InputSpec{metadataKeyInput(0)} + } + if path[1] == "indexes" { + return []pluginsdk.InputSpec{{Title: "Canonical index ID", Description: indexIDHelp + " Use ledger indexes list to inspect registered IDs.", Kind: "input", Argument: new(0), ValueType: "string", Required: true}} + } + + return nil +} + +func resourceInput(collection string) pluginsdk.InputSpec { + input := pluginsdk.InputSpec{Title: "Account address", Kind: "select", Argument: new(0), ValueType: "string", Required: true, + Source: &pluginsdk.ChoiceSource{CommandPath: []string{"ledger", collection, "list"}, Flags: map[string]string{"ledger": "$ledger"}, ValueField: "address", AfterField: "address", LabelFields: []string{"address"}, EmptyMessage: "No accounts are available in this ledger"}} + if collection == "transactions" { + input.Title = "Transaction ID" + // IDs are unsigned integers, but the argument is an exact decimal + // string. The host must keep JSON numbers as json.Number. + input.Source.ValueField = "id" + input.Source.AfterField = "id" + input.Source.LabelFields = []string{"id", "reference"} + input.Source.EmptyMessage = "No transactions are available in this ledger" + } + + return input +} + +func metadataKeyInput(argument int) pluginsdk.InputSpec { + return pluginsdk.InputSpec{Title: "Metadata key", Description: "Exact key to delete", Kind: "input", Argument: new(argument), ValueType: "string", Required: true} +} + +const indexIDHelp = "Canonical ID, e.g. log_builtin:LOG_BUILTIN_INDEX_DATE, tx_builtin:TX_BUILTIN_INDEX_REFERENCE, account_builtin:ACCT_BUILTIN_INDEX_ASSET, metadata:TARGET_TYPE_ACCOUNT:color or metadata:TARGET_TYPE_TRANSACTION:order_id. Metadata fields must have a declared schema type." diff --git a/misc/fctl-plugin/interaction_test.go b/misc/fctl-plugin/interaction_test.go new file mode 100644 index 0000000000..61aa1d00b2 --- /dev/null +++ b/misc/fctl-plugin/interaction_test.go @@ -0,0 +1,377 @@ +package ledger + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "slices" + "strings" + "testing" + + "github.com/formancehq/fctl/pkg/pluginsdk" +) + +func TestLedgerInteractionCoverage(t *testing.T) { + t.Parallel() + m, operations := buildLayout() + if m.Root.Target != "stack" { + t.Fatal("Ledger must target stack context") + } + walkLedgerInputs(t, m.Root) + for path, op := range operations { + command, err := pluginsdk.FindCommand(m, strings.Split(path, "/")) + if err != nil { + t.Fatal(err) + } + checkLedgerTargetInput(t, command, op) + checkArgumentCoverage(t, command, op) + for _, input := range command.Inputs { + checkLedgerInputContract(t, m, command, input) + } + } +} + +func walkLedgerInputs(t *testing.T, command pluginsdk.CommandSpec) { + t.Helper() + if !command.Runnable && len(command.Inputs) != 0 { + t.Fatalf("group %s declares inherited inputs", command.Use) + } + for _, child := range command.Subcommands { + walkLedgerInputs(t, child) + } +} + +func checkLedgerTargetInput(t *testing.T, command pluginsdk.CommandSpec, op operation) { + t.Helper() + index := slices.IndexFunc(command.Inputs, func(input pluginsdk.InputSpec) bool { return input.Flag == "ledger" }) + if op.global { + if index >= 0 { + t.Fatalf("global %s asks for a ledger", command.Use) + } + + return + } + if index != 0 || !command.Inputs[index].Required { + t.Fatalf("%s must resolve the ledger first", command.Use) + } + input := command.Inputs[index] + if op.ledgerArg && (input.AlternativeArgument == nil || *input.AlternativeArgument != 0) { + t.Fatalf("%s lost positional ledger binding", command.Use) + } + if op.ledgerArg && op.method == http.MethodPost { + if input.Kind != "input" || input.Title != "Ledger name" || input.Source != nil { + t.Fatal("ledger creation must ask for a new name") + } + + return + } + if input.Kind != "select" || input.Source == nil || input.Source.ValueField != "name" || !slices.Equal(input.Source.CommandPath, []string{"ledger", "list"}) { + t.Fatalf("invalid ledger selector: %#v", input) + } +} + +func checkArgumentCoverage(t *testing.T, command pluginsdk.CommandSpec, op operation) { + t.Helper() + if op.ledgerArg { + return + } + for i := range op.args { + index := slices.IndexFunc(command.Inputs, func(input pluginsdk.InputSpec) bool { return input.Argument != nil && *input.Argument == i }) + if index < 0 || !command.Inputs[index].Required { + t.Fatalf("%s is missing required argument %d", command.Use, i) + } + } +} + +func checkLedgerInputContract(t *testing.T, m pluginsdk.Manifest, command pluginsdk.CommandSpec, input pluginsdk.InputSpec) { + t.Helper() + bindings := 0 + for _, bound := range []bool{input.Flag != "", input.Argument != nil, input.BodyPointer != "", input.Context != ""} { + if bound { + bindings++ + } + } + if bindings != 1 || input.Title == "" || !slices.Contains([]string{"input", "text", "select", "confirm"}, input.Kind) || !slices.Contains([]string{"", "string", "json", "bool", "number"}, input.ValueType) { + t.Fatalf("invalid input: %#v", input) + } + if input.Argument != nil && (*input.Argument < 0 || *input.Argument >= command.Args.Max) { + t.Fatalf("argument binding out of range: %#v", input) + } + if input.Flag != "" && input.Flag != "ledger" && !slices.ContainsFunc(command.Flags, func(flag pluginsdk.FlagSpec) bool { return flag.Name == input.Flag }) { + t.Fatalf("unknown flag binding: %#v", input) + } + if input.BodyPointer != "" { + checkLedgerBodyPointer(t, input.BodyPointer) + } + if input.Source != nil { + checkLedgerSourceContract(t, m, input.Source) + } +} + +func checkLedgerBodyPointer(t *testing.T, pointer string) { + t.Helper() + if !strings.HasPrefix(pointer, "/") { + t.Fatalf("invalid JSON pointer %q", pointer) + } + for i := 0; i < len(pointer); i++ { + if pointer[i] == '~' { + i++ + if i >= len(pointer) || (pointer[i] != '0' && pointer[i] != '1') { + t.Fatalf("invalid JSON pointer escape %q", pointer) + } + } + } +} + +func checkLedgerSourceContract(t *testing.T, m pluginsdk.Manifest, source *pluginsdk.ChoiceSource) { + t.Helper() + command, err := pluginsdk.FindCommand(m, source.CommandPath) + if err != nil || !command.Runnable || command.Confirm || len(source.Args) != command.Args.Min || source.ValueField == "" || source.EmptyMessage == "" { + t.Fatalf("invalid source: %#v, %v", source, err) + } + _, operations := buildLayout() + if operations[strings.Join(source.CommandPath, "/")].method != http.MethodGet { + t.Fatalf("source can mutate resources: %#v", source) + } + continuationFields := map[string]string{ + "ledger/accounts/list": "address", + "ledger/transactions/list": "id", + } + if source.AfterField != continuationFields[strings.Join(source.CommandPath, "/")] { + t.Fatalf("incorrect pagination descriptor: %#v", source) + } + for flag, value := range source.Flags { + if flag != "ledger" || value != "$ledger" { + t.Fatalf("unexpected source flag %s=%s", flag, value) + } + } +} + +type ledgerChoiceCase struct { + command, field, response, want, path string +} + +func TestLedgerChoiceSourcesPreserveExactValues(t *testing.T) { + t.Parallel() + for _, tc := range []ledgerChoiceCase{ + {"show", "name", `{"data":[{"name":"books"}]}`, "books", "/v3/"}, + {"accounts show", "address", `{"data":[{"address":"users:a/b% c"}]}`, "users:a/b% c", "/v3/books/accounts"}, + {"transactions show", "id", `{"data":[{"id":18446744073709551615,"reference":"payment-42"}]}`, "18446744073709551615", "/v3/books/transactions"}, + {"transactions show", "id", `{"data":[{"id":"18446744073709551615","reference":"payment-42"}]}`, "18446744073709551615", "/v3/books/transactions"}, + } { + t.Run(tc.command, func(t *testing.T) { checkLedgerChoice(t, tc) }) + } +} + +func checkLedgerChoice(t *testing.T, tc ledgerChoiceCase) { + t.Helper() + command, err := pluginsdk.FindCommand(manifestForInputs(), strings.Fields("ledger "+tc.command)) + if err != nil { + t.Fatal(err) + } + index := slices.IndexFunc(command.Inputs, func(input pluginsdk.InputSpec) bool { + return input.Source != nil && input.Source.ValueField == tc.field + }) + if index < 0 { + t.Fatal("missing choice source") + } + result := executeLedgerChoice(t, command.Inputs[index].Source, tc.path, tc.response) + decoder := json.NewDecoder(bytes.NewReader(result)) + decoder.UseNumber() + var envelope struct { + Data []map[string]any `json:"data"` + } + if err := decoder.Decode(&envelope); err != nil || len(envelope.Data) != 1 { + t.Fatalf("source result=%s, error=%v", result, err) + } + value := exactChoiceValue(t, envelope.Data[0][tc.field]) + if value != tc.want { + t.Fatalf("choice value=%q, want %q", value, tc.want) + } + if tc.field == "id" && validateArgs(operation{transactionID: true}, []string{value}) != nil { + t.Fatal("exact selected transaction ID is invalid") + } +} + +func manifestForInputs() pluginsdk.Manifest { + m, _ := buildLayout() + + return m +} + +func executeLedgerChoice(t *testing.T, source *pluginsdk.ChoiceSource, path, body string) json.RawMessage { + t.Helper() + calls := 0 + p := New(&http.Client{Transport: transportFunc(func(req *http.Request) (*http.Response, error) { + calls++ + if req.Method != http.MethodGet || req.URL.Path != "/gateway/ledger"+path || req.Context() != t.Context() { + t.Fatalf("choice request=%s %s", req.Method, req.URL) + } + + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}, nil + })}) + flags := make(map[string]string) + for flag, value := range source.Flags { + if value != "$ledger" { + t.Fatalf("unknown substitution %q", value) + } + flags[flag] = "books" + } + result, err := p.Execute(t.Context(), pluginsdk.ExecuteRequest{CommandPath: source.CommandPath, Args: source.Args, Flags: flags, Endpoint: "https://example.test/gateway/ledger"}) + if err != nil || calls != 1 { + t.Fatalf("source calls=%d, error=%v", calls, err) + } + + return result.Data +} + +func exactChoiceValue(t *testing.T, value any) string { + t.Helper() + switch value := value.(type) { + case string: + return value + case json.Number: + return value.String() + default: + t.Fatalf("unsupported choice value %T", value) + + return "" + } +} + +func TestLedgerStructuredBodyAndAdvancedJSONInputs(t *testing.T) { + t.Parallel() + m := manifestForInputs() + command, err := pluginsdk.FindCommand(m, strings.Fields("ledger transactions create")) + if err != nil { + t.Fatal(err) + } + fields := map[string]pluginsdk.InputSpec{} + for _, input := range command.Inputs { + if input.BodyPointer != "" { + fields[input.BodyPointer] = input + } + } + if len(fields) != 4 || fields["/script/plain"].Kind != "text" || !fields["/script/plain"].Required || !strings.Contains(fields["/script/plain"].Default, "send [USD/2 100]") { + t.Fatalf("invalid Numscript form: %#v", fields) + } + if fields["/reference"].Required || fields["/metadata"].Required || fields["/metadata"].Default != "" || fields["/metadata"].ValueType != "json" { + t.Fatal("optional transaction fields cannot be forced") + } + if fields["/force"].Kind != "confirm" || fields["/force"].ValueType != "bool" || fields["/force"].Default != "false" { + t.Fatal("force must be an explicit boolean choice") + } + for _, path := range []string{"ledger metadata set", "ledger accounts metadata set", "ledger transactions metadata set", "ledger bulk"} { + checkAdvancedJSONInput(t, m, path) + } + index, err := pluginsdk.FindCommand(m, strings.Fields("ledger indexes create")) + if err != nil { + t.Fatal(err) + } + input := index.Inputs[1] + if input.BodyPointer != "/id" || input.Kind != "input" || input.Default != "log_builtin:LOG_BUILTIN_INDEX_DATE" || input.Source != nil { + t.Fatal("new indexes must accept a canonical ID that does not exist yet") + } +} + +func checkAdvancedJSONInput(t *testing.T, m pluginsdk.Manifest, path string) { + t.Helper() + command, err := pluginsdk.FindCommand(m, strings.Fields(path)) + if err != nil { + t.Fatal(err) + } + index := slices.IndexFunc(command.Inputs, func(input pluginsdk.InputSpec) bool { return input.Flag == "data" }) + if index < 0 { + t.Fatalf("%s has no guided JSON editor", path) + } + input := command.Inputs[index] + if input.Kind != "text" || input.ValueType != "json" || !input.Required || !json.Valid([]byte(input.Default)) { + t.Fatalf("invalid JSON input: %#v", input) + } + if path == "ledger bulk" && !strings.HasPrefix(input.Default, "[") { + t.Fatal("bulk requires an operation array") + } + if path != "ledger bulk" && !strings.HasPrefix(input.Default, "{") { + t.Fatal("metadata requires a JSON object") + } +} + +func TestLedgerCreationSettingsMatchV3Contract(t *testing.T) { + t.Parallel() + command, err := pluginsdk.FindCommand(manifestForInputs(), []string{"ledger", "create"}) + if err != nil { + t.Fatal(err) + } + if len(command.Inputs) != 5 { + t.Fatalf("creation form inputs=%d, want name and four settings", len(command.Inputs)) + } + enforcement := command.Inputs[1] + if enforcement.BodyPointer != "/defaultEnforcementMode" || enforcement.Kind != "select" || enforcement.Default != "STRICT" { + t.Fatalf("incorrect server default: %#v", enforcement) + } + values := make([]string, 0, len(enforcement.Options)) + for _, option := range enforcement.Options { + values = append(values, option.Value) + } + if !slices.Equal(values, []string{"STRICT", "AUDIT"}) { + t.Fatalf("unsupported enforcement modes: %v", values) + } + for i, pointer := range []string{"/metadata", "/initialSchema", "/accountTypes"} { + input := command.Inputs[i+2] + if input.BodyPointer != pointer || input.Kind != "text" || input.ValueType != "json" || input.Required || input.Default != "" { + t.Fatalf("optional creation setting must stay omitted when blank: %#v", input) + } + } + // This payload follows the release/v3.0 HTTP createLedgerBody and its + // initial-metadata fixture. Integer metadata is never decoded as float64. + body := `{"defaultEnforcementMode":"STRICT","metadata":{"count":9007199254740993,"purpose":"tests","enabled":true},"initialSchema":[{"targetType":"account","key":"color","type":"string"}],"accountTypes":{"user-checking":{"name":"user-checking","pattern":"users:{id}:checking","persistence":"EPHEMERAL","segmentTypes":{"id":{"type":"uint64"}}}}}` + calls := 0 + p := New(&http.Client{Transport: transportFunc(func(req *http.Request) (*http.Response, error) { + calls++ + data, err := io.ReadAll(req.Body) + if err != nil || string(data) != body || req.Method != http.MethodPost || req.URL.Path != "/v3/books" { + t.Fatalf("create request=%s %s body=%s, error=%v", req.Method, req.URL, data, err) + } + + return &http.Response{StatusCode: http.StatusCreated, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"data":{"name":"books"}}`))}, nil + })}) + _, err = p.Execute(t.Context(), pluginsdk.ExecuteRequest{CommandPath: []string{"ledger", "create"}, Args: []string{"books"}, Body: json.RawMessage(body), Endpoint: "https://example.test"}) + if err != nil || calls != 1 { + t.Fatalf("create calls=%d, error=%v", calls, err) + } +} + +func TestV3ListResponseWithoutPaginationEnvelope(t *testing.T) { + t.Parallel() + for _, tc := range []ledgerChoiceCase{ + {"accounts show", "address", `{"data":[{"address":"users:next"}]}`, "users:next", "/v3/books/accounts"}, + {"transactions show", "id", `{"data":[{"id":9007199254740993}]}`, "9007199254740993", "/v3/books/transactions"}, + } { + t.Run(tc.command, func(t *testing.T) { + m := manifestForInputs() + command, err := pluginsdk.FindCommand(m, strings.Fields("ledger "+tc.command)) + if err != nil { + t.Fatal(err) + } + source := command.Inputs[1].Source + data := executeLedgerChoice(t, source, tc.path, tc.response) + var envelope map[string]json.RawMessage + if err := json.Unmarshal(data, &envelope); err != nil || len(envelope) != 1 || envelope["data"] == nil { + t.Fatalf("v3 list envelope=%s, error=%v", data, err) + } + request, err := pluginsdk.NormalizeRequest(m, pluginsdk.ExecuteRequest{CommandPath: source.CommandPath, Flags: map[string]string{"ledger": "books", "after": tc.want}}) + if err != nil || request.Flags["after"] != tc.want { + t.Fatalf("last item cannot be used as after=%s: %v", tc.want, err) + } + cursor := base64.RawURLEncoding.EncodeToString([]byte(`{"key":"` + tc.want + `"}`)) + request, err = pluginsdk.NormalizeRequest(m, pluginsdk.ExecuteRequest{CommandPath: source.CommandPath, Flags: map[string]string{"ledger": "books", "cursor": cursor}}) + if err != nil || request.Flags["cursor"] != cursor { + t.Fatalf("entity list rejected a page cursor: request=%#v err=%v", request, err) + } + tokens := append([]string{"--ledger", "books"}, source.CommandPath[1:]...) + assertPaginationRejected(t, append(tokens, "--cursor", tc.want), "invalid --cursor") + }) + } +} diff --git a/misc/fctl-plugin/internal/catalogue/publish.go b/misc/fctl-plugin/internal/catalogue/publish.go new file mode 100644 index 0000000000..d0d6a2adf9 --- /dev/null +++ b/misc/fctl-plugin/internal/catalogue/publish.go @@ -0,0 +1,288 @@ +package catalogue + +import ( + "context" + "debug/buildinfo" + "encoding/json" + "errors" + "fmt" + "io" + "net/url" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "github.com/formancehq/fctl/pkg/pluginsdk" +) + +type PublishOptions struct { + ServiceVersion string + ManifestPath string + ArtifactsPath string + SourceRoot string + Registry string + Repository string + Revision int + Layout string +} + +// buildArtifact is the GoReleaser artifacts.json contract used by its Go builds. +type buildArtifact struct { + Path string `json:"path"` + OS string `json:"goos"` + Arch string `json:"goarch"` + Type string `json:"type"` + Extra struct { + ID string `json:"ID"` + } `json:"extra"` +} + +// Publish uploads raw GoReleaser executables and emits the catalogue only after +// every platform succeeds. Layout writes real OCI artifacts locally, without +// contacting a registry. Diagnostics never share the catalogue output stream. +func Publish(ctx context.Context, options PublishOptions, output, diagnostics io.Writer) (err error) { + if options.Layout != "" { + options.Layout, err = filepath.Abs(options.Layout) + if err != nil { + return err + } + } + stagePath, err := os.MkdirTemp("", "fctl-plugin-publish-") + if err != nil { + return err + } + defer func() { err = errors.Join(err, os.RemoveAll(stagePath)) }() + releases, err := prepare(options, stagePath) + if err != nil { + return err + } + origin, err := url.Parse(options.Registry) + if err != nil { + return err + } + tags, err := releaseTags(releases) + if err != nil { + return err + } + for index, release := range releases { + digest, err := push(ctx, options, origin, stagePath, release, tags[index], diagnostics) + if err != nil { + return err + } + releases[index].Artifact.Digest = digest + if err := validateArtifact(releases[index].Artifact); err != nil { + return err + } + } + + return json.NewEncoder(output).Encode(Catalogue{SchemaVersion: 1, Releases: releases}) +} + +// prepare freezes and validates every binary before any registry operation. +func prepare(options PublishOptions, stagePath string) (releases []Release, err error) { + manifest, err := ReadManifest(options.ManifestPath) + if err != nil { + return nil, err + } + if options.ServiceVersion == "" || manifest.Version != options.ServiceVersion { + return nil, errors.New("manifest service version must match the exact release service version") + } + data, err := os.ReadFile(options.ArtifactsPath) // #nosec G304 -- Explicit release-tool input, not plugin metadata. + if err != nil { + return nil, err + } + artifacts, err := readArtifacts(data) + if err != nil { + return nil, err + } + + sourceRoot, err := os.OpenRoot(options.SourceRoot) + if err != nil { + return nil, err + } + defer func() { err = errors.Join(err, sourceRoot.Close()) }() + sourcePath, err := filepath.Abs(options.SourceRoot) + if err != nil { + return nil, err + } + stageRoot, err := os.OpenRoot(stagePath) + if err != nil { + return nil, err + } + defer func() { err = errors.Join(err, stageRoot.Close()) }() + + return prepareArtifacts(options, manifest, artifacts, sourceRoot, sourcePath, stageRoot, stagePath) +} + +func readArtifacts(data []byte) ([]buildArtifact, error) { + var artifacts []buildArtifact + if err := json.Unmarshal(data, &artifacts); err != nil { + return nil, fmt.Errorf("read GoReleaser artifacts: %w", err) + } + + return artifacts, nil +} + +func prepareArtifacts(options PublishOptions, manifest pluginsdk.Manifest, artifacts []buildArtifact, sourceRoot *os.Root, sourcePath string, stageRoot *os.Root, stagePath string) ([]Release, error) { + var releases []Release + seen := make(map[Platform]bool) + for _, artifact := range artifacts { + if artifact.Type != "Binary" || artifact.Extra.ID != "fctl-plugin-ledger" { + continue + } + platform := Platform{OS: artifact.OS, Arch: artifact.Arch} + if seen[platform] { + return nil, fmt.Errorf("duplicate binary for %s/%s", platform.OS, platform.Arch) + } + seen[platform] = true + name, err := stageBinary(sourceRoot, sourcePath, stageRoot, artifact, manifest.Version, options.Revision) + if err != nil { + return nil, err + } + release, err := NewRelease(manifest, filepath.Join(stagePath, name), + Artifact{Registry: options.Registry, Repository: options.Repository, Digest: "sha256:" + strings.Repeat("0", 64)}, + platform, options.Revision) + if err != nil { + return nil, err + } + releases = append(releases, release) + } + if len(releases) != 6 { + return nil, fmt.Errorf("expected six GoReleaser plugin platforms, got %d", len(releases)) + } + + return releases, nil +} + +func releaseTags(releases []Release) ([]string, error) { + tags := make([]string, len(releases)) + for index, release := range releases { + tag := fmt.Sprintf("%s-r%d-%s-%s-%s", strings.ReplaceAll(release.ServiceVersion, "+", "_"), release.Revision, + release.Platform.OS, release.Platform.Arch, release.SHA256[:16]) + if len(tag) > 128 || strings.ContainsAny(tag, ":@") { + return nil, errors.New("service version produces an invalid OCI tag") + } + tags[index] = tag + } + + return tags, nil +} + +func push(ctx context.Context, options PublishOptions, origin *url.URL, stagePath string, release Release, tag string, diagnostics io.Writer) (string, error) { + destination := origin.Host + "/" + options.Repository + ":" + tag + if options.Layout != "" { + destination = options.Layout + ":" + tag + } + args := []string{"push", destination, "fctl-plugin-ledger:application/vnd.formance.fctl.plugin.executable.v1", + "--artifact-type", "application/vnd.formance.fctl.plugin.v1", "--image-spec", "v1.1", + "--config", "empty.json:application/vnd.oci.empty.v1+json", "--format", "json", + "--annotation", "org.opencontainers.image.source=https://github.com/formancehq/ledger"} + if options.Layout != "" { + args = append(args, "--oci-layout") + } else if origin.Scheme == "http" { + args = append(args, "--plain-http") + } + command := exec.CommandContext(ctx, "oras", args...) // #nosec G204 -- Fixed executable and validated identity; no shell. + command.Dir = filepath.Join(stagePath, release.Platform.OS+"_"+release.Platform.Arch) + command.Stderr = diagnostics + result, err := command.Output() + if err != nil { + return "", fmt.Errorf("publish %s/%s: %w", release.Platform.OS, release.Platform.Arch, err) + } + var pushed struct { + Digest string `json:"digest"` + } + if err := json.Unmarshal(result, &pushed); err != nil { + return "", fmt.Errorf("read ORAS push digest: %w", err) + } + + return pushed.Digest, nil +} + +func stageBinary(sourceRoot *os.Root, sourcePath string, stageRoot *os.Root, artifact buildArtifact, version string, revision int) (name string, err error) { + if artifact.OS != "linux" && artifact.OS != "darwin" && artifact.OS != "windows" || artifact.Arch != "amd64" && artifact.Arch != "arm64" { + return "", errors.New("unsupported GoReleaser platform") + } + relative := artifact.Path + if filepath.IsAbs(relative) { + var err error + relative, err = filepath.Rel(sourcePath, relative) + if err != nil { + return "", err + } + } + if !filepath.IsLocal(relative) { + return "", errors.New("GoReleaser executable path must remain inside its build root") + } + source, err := sourceRoot.Open(relative) + if err != nil { + return "", fmt.Errorf("open contained GoReleaser executable: %w", err) + } + defer func() { err = errors.Join(err, source.Close()) }() + if err := validateBinary(source, artifact, version, revision); err != nil { + return "", err + } + + directory := artifact.OS + "_" + artifact.Arch + if err := stageRoot.Mkdir(directory, 0o700); err != nil { + return "", err + } + if err := stageRoot.WriteFile(filepath.Join(directory, "empty.json"), []byte("{}"), 0o600); err != nil { + return "", err + } + name = filepath.Join(directory, "fctl-plugin-ledger") + target, err := stageRoot.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return "", err + } + _, copyErr := io.Copy(target, source) + if err := errors.Join(copyErr, target.Close()); err != nil { + return "", err + } + + return name, nil +} + +func hasLinkValue(flags, name, value string) bool { + words := strings.Fields(flags) + var actual string + for index, word := range words { + if word == "-X" && index+1 < len(words) { + if field, ok := strings.CutPrefix(words[index+1], name+"="); ok { + actual = field + } + } + } + + return actual == value +} + +func validateBinary(source *os.File, artifact buildArtifact, version string, revision int) error { + info, err := source.Stat() + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() < 1 || info.Size() > maxBinaryBytes { + return errors.New("GoReleaser executable must be a regular file containing 1 byte to 128 MiB") + } + build, err := buildinfo.Read(source) + if err != nil { + return fmt.Errorf("read Go executable build identity: %w", err) + } + settings := make(map[string]string) + for _, setting := range build.Settings { + settings[setting.Key] = setting.Value + } + if settings["GOOS"] != artifact.OS || settings["GOARCH"] != artifact.Arch || + build.Path != "github.com/formancehq/ledger/misc/fctl-plugin/cmd/fctl-plugin-ledger" { + return errors.New("GoReleaser binary platform or product entry point does not match catalogue inputs") + } + if !hasLinkValue(settings["-ldflags"], "main.serviceVersion", version) || + !hasLinkValue(settings["-ldflags"], "main.revision", strconv.Itoa(revision)) { + return errors.New("GoReleaser binary service version or plugin revision does not match catalogue inputs; build without -trimpath to retain linker metadata") + } + + return nil +} diff --git a/misc/fctl-plugin/internal/catalogue/publish_failure_it_test.go b/misc/fctl-plugin/internal/catalogue/publish_failure_it_test.go new file mode 100644 index 0000000000..29a40e8c84 --- /dev/null +++ b/misc/fctl-plugin/internal/catalogue/publish_failure_it_test.go @@ -0,0 +1,284 @@ +//go:build it + +package catalogue + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/formancehq/fctl/pkg/pluginsdk" + + ledger "github.com/formancehq/ledger/misc/fctl-plugin" +) + +func TestPublisherPreflightAndFailureMatrix(t *testing.T) { + t.Parallel() + if _, err := exec.LookPath("oras"); err != nil { + t.Fatal("run publisher integration tests in the repository-pinned Nix environment: ", err) + } + buildDirectory := t.TempDir() + artifacts := buildPlatforms(t, buildDirectory) + manifest, err := ledger.NewVersion(nil, "3.0.0-beta.10").GetManifest(t.Context()) + if err != nil { + t.Fatal(err) + } + manifestPath := filepath.Join(buildDirectory, "manifest.json") + writeFailureFixtureJSON(t, manifestPath, manifest) + artifactsPath := filepath.Join(buildDirectory, "artifacts.json") + writeFailureFixtureJSON(t, artifactsPath, artifacts) + layout := filepath.Join(t.TempDir(), "oci") + options := PublishOptions{ServiceVersion: manifest.Version, ManifestPath: manifestPath, ArtifactsPath: artifactsPath, SourceRoot: buildDirectory, + Registry: "http://127.0.0.1:5000", Repository: "formancehq/fctl-plugin-ledger", Revision: 2, Layout: layout} + for _, failure := range []string{"missing service version", "release tag not stripped", "stale manifest", "wrong manifest product", "wrong manifest service", "wrong manifest command", "wrong platform", "wrong revision", "mixed service versions", "mixed plugin revisions", "wrong entry point", "missing platform", "duplicate platform"} { + t.Run(failure, func(t *testing.T) { + invalid, expectedError := invalidPublishOptions(t, failure, options, manifest, artifacts, buildDirectory) + assertPreflightRejected(t, invalid, expectedError) + }) + } + t.Run("ORAS invocation failure emits no catalogue", func(t *testing.T) { + assertORASFailure(t, options) + }) + var output bytes.Buffer + if err := Publish(t.Context(), options, &output, io.Discard); err != nil { + t.Fatal(err) + } + var catalogue Catalogue + if err := json.Unmarshal(output.Bytes(), &catalogue); err != nil || catalogue.SchemaVersion != 1 || len(catalogue.Releases) != 6 { + t.Fatalf("catalogue=%s, err=%v", output.Bytes(), err) + } + verifyLayout(t, layout, buildDirectory, options.ServiceVersion, catalogue) +} + +func writeFailureFixtureJSON(t *testing.T, path string, value any) { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } +} + +func buildPlatforms(t *testing.T, buildDirectory string) []buildArtifact { + t.Helper() + var artifacts []buildArtifact + for _, goos := range []string{"linux", "darwin", "windows"} { + for _, arch := range []string{"amd64", "arm64"} { + path := filepath.Join(buildDirectory, goos+"_"+arch) + buildPlugin(t, path, goos, arch, "-X main.serviceVersion=3.0.0-beta.10 -X main.revision=2", "./cmd/fctl-plugin-ledger") + artifact := buildArtifact{Path: path, OS: goos, Arch: arch, Type: "Binary"} + artifact.Extra.ID = "fctl-plugin-ledger" + artifacts = append(artifacts, artifact) + } + } + + return artifacts +} + +func invalidPublishOptions(t *testing.T, failure string, options PublishOptions, manifest pluginsdk.Manifest, artifacts []buildArtifact, buildDirectory string) (PublishOptions, string) { + t.Helper() + invalid := options + invalid.Layout = filepath.Join(t.TempDir(), "must-not-publish") + expectedError := "service version or plugin revision" + switch failure { + case "missing service version": + invalid.ServiceVersion = "" + expectedError = "service version" + case "release tag not stripped": + invalid.ServiceVersion = "v" + manifest.Version + expectedError = "service version" + case "stale manifest": + stale := manifest + stale.Version = "3.0.999" + invalid.ManifestPath = filepath.Join(t.TempDir(), "stale.json") + writeFailureFixtureJSON(t, invalid.ManifestPath, stale) + expectedError = "service version" + case "wrong manifest product", "wrong manifest service", "wrong manifest command": + wrong := manifest + switch failure { + case "wrong manifest product": + wrong.Name = "auth" + case "wrong manifest service": + wrong.Service = "auth" + case "wrong manifest command": + wrong.Root.Use = "auth" + } + invalid.ManifestPath = filepath.Join(t.TempDir(), "wrong-product.json") + writeFailureFixtureJSON(t, invalid.ManifestPath, wrong) + expectedError = "manifest must describe" + case "wrong platform": + mixed := append([]buildArtifact{}, artifacts...) + // The last binary claims windows/arm64 but contains linux/amd64. + mixed[len(mixed)-1].Path = artifacts[0].Path + invalid.ArtifactsPath = filepath.Join(t.TempDir(), "wrong-platform.json") + writeFailureFixtureJSON(t, invalid.ArtifactsPath, mixed) + expectedError = "platform or product entry point" + case "missing platform": + invalid.ArtifactsPath = filepath.Join(t.TempDir(), "missing-platform.json") + writeFailureFixtureJSON(t, invalid.ArtifactsPath, artifacts[:len(artifacts)-1]) + expectedError = "expected six" + case "duplicate platform": + mixed := append([]buildArtifact{}, artifacts...) + mixed = append(mixed, artifacts[0]) + invalid.ArtifactsPath = filepath.Join(t.TempDir(), "duplicate-platform.json") + writeFailureFixtureJSON(t, invalid.ArtifactsPath, mixed) + expectedError = "duplicate binary" + case "wrong revision": + invalid.Revision = 99 + case "mixed service versions", "mixed plugin revisions": + path := filepath.Join(buildDirectory, strings.ReplaceAll(failure, " ", "-")+"-windows-arm64") + linkFlags := "-X main.serviceVersion=3.0.1 -X main.revision=2" + if failure == "mixed plugin revisions" { + linkFlags = "-X main.serviceVersion=3.0.0-beta.10 -X main.revision=99" + } + buildPlugin(t, path, "windows", "arm64", linkFlags, "./cmd/fctl-plugin-ledger") + mixed := append([]buildArtifact{}, artifacts...) + mixed[len(mixed)-1].Path = path + invalid.ArtifactsPath = filepath.Join(t.TempDir(), "mixed.json") + writeFailureFixtureJSON(t, invalid.ArtifactsPath, mixed) + case "wrong entry point": + path := filepath.Join(buildDirectory, "wrong-entry-point") + buildPlugin(t, path, "windows", "arm64", "-X main.serviceVersion=3.0.0-beta.10 -X main.revision=2", "./cmd/fctl-plugin-publish") + mixed := append([]buildArtifact{}, artifacts...) + mixed[len(mixed)-1].Path = path + invalid.ArtifactsPath = filepath.Join(t.TempDir(), "wrong-entry-point.json") + writeFailureFixtureJSON(t, invalid.ArtifactsPath, mixed) + expectedError = "product entry point" + } + + return invalid, expectedError +} + +func assertORASFailure(t *testing.T, options PublishOptions) { + t.Helper() + invalid := options + // A file cannot contain an OCI layout. Real ORAS must fail locally. + invalid.Layout = filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(invalid.Layout, []byte("block layout creation"), 0o600); err != nil { + t.Fatal(err) + } + var rejected, diagnostics bytes.Buffer + err := Publish(t.Context(), invalid, &rejected, &diagnostics) + if err == nil || !strings.Contains(err.Error(), "publish linux/amd64") { + t.Fatalf("expected ORAS invocation failure, got %v (diagnostics: %s)", err, &diagnostics) + } + if rejected.Len() != 0 { + t.Fatalf("failed ORAS advertised a catalogue: %s", &rejected) + } +} + +func verifyLayout(t *testing.T, layout, buildDirectory, version string, catalogue Catalogue) { + t.Helper() + root, err := os.OpenRoot(layout) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := root.Close(); err != nil { + t.Error(err) + } + }) + source := openTestRoot(t, buildDirectory) + for _, release := range catalogue.Releases { + verifyRelease(t, root, source, version, release) + } +} + +func verifyRelease(t *testing.T, root, source *os.Root, version string, release Release) { + t.Helper() + digest := strings.TrimPrefix(release.Artifact.Digest, "sha256:") + data, err := root.ReadFile(filepath.Join("blobs", "sha256", digest)) + if err != nil { + t.Fatal(err) + } + checksum := sha256.Sum256(data) + if hex.EncodeToString(checksum[:]) != digest || release.ServiceVersion != "3.0.0-beta.10" || release.Revision != 2 { + t.Fatalf("invalid immutable release identity: %#v", release) + } + verifyOCIManifest(t, data, release.SHA256) + binary, err := root.ReadFile(filepath.Join("blobs", "sha256", release.SHA256)) + if err != nil { + t.Fatal(err) + } + original, err := source.ReadFile(release.Platform.OS + "_" + release.Platform.Arch) + if err != nil { + t.Fatal(err) + } + checksum = sha256.Sum256(binary) + if !bytes.Equal(binary, original) || hex.EncodeToString(checksum[:]) != release.SHA256 || + release.Service != "ledger" || release.Manifest.Name != "ledger" || release.Manifest.Version != version { + t.Fatalf("OCI executable or Ledger manifest differs from release input: %#v", release) + } +} + +func verifyOCIManifest(t *testing.T, data []byte, checksum string) { + t.Helper() + var image struct { + MediaType string `json:"mediaType"` + ArtifactType string `json:"artifactType"` + Config struct { + MediaType string `json:"mediaType"` + } `json:"config"` + Layers []struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + } `json:"layers"` + } + if err := json.Unmarshal(data, &image); err != nil { + t.Fatal(err) + } + if image.MediaType != "application/vnd.oci.image.manifest.v1+json" || image.ArtifactType != "application/vnd.formance.fctl.plugin.v1" || + image.Config.MediaType != "application/vnd.oci.empty.v1+json" || len(image.Layers) != 1 || + image.Layers[0].MediaType != "application/vnd.formance.fctl.plugin.executable.v1" || image.Layers[0].Digest != "sha256:"+checksum { + t.Fatalf("unexpected OCI manifest: %s", data) + } +} + +func buildPlugin(t *testing.T, path, goos, arch, linkFlags, entryPoint string) { + t.Helper() + //nolint:gosec // Arguments are controlled test fixtures; go runs directly without a shell. + command := exec.CommandContext(t.Context(), "go", "build", "-o", path, "-ldflags", linkFlags, entryPoint) + command.Dir = "../.." + command.Env = append(os.Environ(), "GOOS="+goos, "GOARCH="+arch, "CGO_ENABLED=0") + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("build %s/%s: %s, %v", goos, arch, output, err) + } +} + +func openTestRoot(t *testing.T, path string) *os.Root { + t.Helper() + root, err := os.OpenRoot(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := root.Close(); err != nil { + t.Error(err) + } + }) + + return root +} + +func assertPreflightRejected(t *testing.T, invalid PublishOptions, expectedError string) { + t.Helper() + var rejected bytes.Buffer + if err := Publish(t.Context(), invalid, &rejected, io.Discard); err == nil || !strings.Contains(err.Error(), expectedError) { + t.Fatalf("wrong release identity accepted or wrong rejection: %v", err) + } + if rejected.Len() != 0 { + t.Fatalf("failed preflight advertised a catalogue: %s", rejected.Bytes()) + } + if _, err := os.Stat(invalid.Layout); !os.IsNotExist(err) { + t.Fatalf("failed preflight reached ORAS publication: %v", err) + } +} diff --git a/misc/fctl-plugin/internal/catalogue/publish_it_test.go b/misc/fctl-plugin/internal/catalogue/publish_it_test.go new file mode 100644 index 0000000000..97d0060fb4 --- /dev/null +++ b/misc/fctl-plugin/internal/catalogue/publish_it_test.go @@ -0,0 +1,145 @@ +//go:build it + +package catalogue + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + ledger "github.com/formancehq/ledger/misc/fctl-plugin" +) + +func TestPublisherCreatesSixRealOCIArtifacts(t *testing.T) { + t.Parallel() + if _, err := exec.LookPath("oras"); err != nil { + t.Fatal("run publisher integration tests in the repository-pinned Nix environment: ", err) + } + buildDirectory := t.TempDir() + var artifacts []buildArtifact + for _, goos := range []string{"linux", "darwin", "windows"} { + for _, arch := range []string{"amd64", "arm64"} { + path := filepath.Join(buildDirectory, goos+"_"+arch) + command := exec.CommandContext(t.Context(), "go", "build", "-o", path, + "-ldflags", "-X main.serviceVersion=3.0.0-beta.10 -X main.revision=2", "./cmd/fctl-plugin-ledger") + command.Dir = "../.." + command.Env = append(os.Environ(), "GOOS="+goos, "GOARCH="+arch, "CGO_ENABLED=0") + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("build %s/%s: %s, %v", goos, arch, output, err) + } + artifact := buildArtifact{Path: path, OS: goos, Arch: arch, Type: "Binary"} + artifact.Extra.ID = "fctl-plugin-ledger" + artifacts = append(artifacts, artifact) + } + } + manifest, err := ledger.NewVersion(nil, "3.0.0-beta.10").GetManifest(t.Context()) + if err != nil { + t.Fatal(err) + } + manifestPath := filepath.Join(buildDirectory, "manifest.json") + writeJSON(t, manifestPath, manifest) + artifactsPath := filepath.Join(buildDirectory, "artifacts.json") + writeJSON(t, artifactsPath, artifacts) + layout := filepath.Join(t.TempDir(), "oci") + options := PublishOptions{ServiceVersion: "3.0.0-beta.10", ManifestPath: manifestPath, ArtifactsPath: artifactsPath, SourceRoot: buildDirectory, + Registry: "http://127.0.0.1:5000", Repository: "formancehq/fctl-plugin-ledger", Revision: 2, Layout: layout} + for _, failure := range []string{"stale manifest", "wrong revision", "mixed service versions"} { + t.Run(failure, func(t *testing.T) { + invalid := options + invalid.Layout = filepath.Join(t.TempDir(), "must-not-publish") + switch failure { + case "stale manifest": + stale := manifest + stale.Version = "3.0.999" + invalid.ManifestPath = filepath.Join(t.TempDir(), "stale.json") + writeJSON(t, invalid.ManifestPath, stale) + case "wrong revision": + invalid.Revision = 99 + case "mixed service versions": + path := filepath.Join(buildDirectory, "wrong-windows-arm64") + command := exec.CommandContext(t.Context(), "go", "build", "-o", path, + "-ldflags", "-X main.serviceVersion=3.0.1 -X main.revision=2", "./cmd/fctl-plugin-ledger") + command.Dir = "../.." + command.Env = append(os.Environ(), "GOOS=windows", "GOARCH=arm64", "CGO_ENABLED=0") + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("build mismatched foreign executable: %s, %v", output, err) + } + mixed := append([]buildArtifact{}, artifacts...) + mixed[len(mixed)-1].Path = path + invalid.ArtifactsPath = filepath.Join(t.TempDir(), "mixed.json") + writeJSON(t, invalid.ArtifactsPath, mixed) + } + var rejected bytes.Buffer + if err := Publish(t.Context(), invalid, &rejected, io.Discard); err == nil || !strings.Contains(err.Error(), "service version") { + t.Fatalf("wrong release identity accepted or wrong rejection: %v", err) + } + if rejected.Len() != 0 { + t.Fatalf("failed preflight advertised a catalogue: %s", rejected.Bytes()) + } + if _, err := os.Stat(invalid.Layout); !os.IsNotExist(err) { + t.Fatalf("failed preflight reached ORAS publication: %v", err) + } + }) + } + var output bytes.Buffer + if err := Publish(t.Context(), options, &output, io.Discard); err != nil { + t.Fatal(err) + } + var catalogue Catalogue + if err := json.Unmarshal(output.Bytes(), &catalogue); err != nil || catalogue.SchemaVersion != 1 || len(catalogue.Releases) != 6 { + t.Fatalf("catalogue=%s, err=%v", output.Bytes(), err) + } + root, err := os.OpenRoot(layout) + if err != nil { + t.Fatal(err) + } + defer func() { _ = root.Close() }() + for _, release := range catalogue.Releases { + digest := strings.TrimPrefix(release.Artifact.Digest, "sha256:") + data, err := root.ReadFile(filepath.Join("blobs", "sha256", digest)) + if err != nil { + t.Fatal(err) + } + checksum := sha256.Sum256(data) + if hex.EncodeToString(checksum[:]) != digest || release.ServiceVersion != "3.0.0-beta.10" || release.Revision != 2 { + t.Fatalf("invalid immutable release identity: %#v", release) + } + var image struct { + MediaType string `json:"mediaType"` + ArtifactType string `json:"artifactType"` + Config struct { + MediaType string `json:"mediaType"` + } `json:"config"` + Layers []struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + } `json:"layers"` + } + if err := json.Unmarshal(data, &image); err != nil { + t.Fatal(err) + } + if image.MediaType != "application/vnd.oci.image.manifest.v1+json" || image.ArtifactType != "application/vnd.formance.fctl.plugin.v1" || + image.Config.MediaType != "application/vnd.oci.empty.v1+json" || len(image.Layers) != 1 || + image.Layers[0].MediaType != "application/vnd.formance.fctl.plugin.executable.v1" || image.Layers[0].Digest != "sha256:"+release.SHA256 { + t.Fatalf("unexpected OCI manifest: %s", data) + } + } +} + +func writeJSON(t *testing.T, path string, value any) { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } +} diff --git a/misc/fctl-plugin/internal/catalogue/publish_test.go b/misc/fctl-plugin/internal/catalogue/publish_test.go new file mode 100644 index 0000000000..c953eb1875 --- /dev/null +++ b/misc/fctl-plugin/internal/catalogue/publish_test.go @@ -0,0 +1,87 @@ +package catalogue + +import ( + "bytes" + "encoding/json" + "io" + "os" + "path/filepath" + "strings" + "testing" + + ledger "github.com/formancehq/ledger/misc/fctl-plugin" +) + +func TestArtifactPathsStayInsideGoReleaserBuildRoot(t *testing.T) { + t.Parallel() + parent := t.TempDir() + buildPath := filepath.Join(parent, "build") + if err := os.Mkdir(buildPath, 0o700); err != nil { + t.Fatal(err) + } + outside := filepath.Join(parent, "outside") + if err := os.WriteFile(outside, []byte("must not publish this file"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(buildPath, "escape")); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(buildPath) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := root.Close(); err != nil { + t.Error(err) + } + }) + stagePath := t.TempDir() + stage, err := os.OpenRoot(stagePath) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := stage.Close(); err != nil { + t.Error(err) + } + }) + for _, path := range []string{outside, "../outside", "escape"} { + artifact := buildArtifact{Path: path, OS: "linux", Arch: "amd64"} + _, err := stageBinary(root, buildPath, stage, artifact, "3.0.0-beta.10", 2) + if err == nil || strings.Contains(err.Error(), "build identity") { + t.Fatalf("path %q reached binary inspection outside root: %v", path, err) + } + } + entries, err := os.ReadDir(stagePath) + if err != nil || len(entries) != 0 { + t.Fatalf("rejected artifact produced staging files: %v, %v", entries, err) + } +} + +func TestPublishRequiresExactServiceVersionBeforeReadingArtifacts(t *testing.T) { + t.Parallel() + manifest, err := ledger.NewVersion(nil, "3.0.0-beta.10").GetManifest(t.Context()) + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "manifest.json") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + for _, version := range []string{"", "3.0.1", "v3.0.0-beta.10"} { + t.Run("version="+version, func(t *testing.T) { + var output bytes.Buffer + options := PublishOptions{ManifestPath: path, ServiceVersion: version, ArtifactsPath: "does-not-exist"} + if err := Publish(t.Context(), options, &output, io.Discard); err == nil || !strings.Contains(err.Error(), "service version") { + t.Fatalf("expected exact service version rejection before reading artifacts, got %v", err) + } + if output.Len() != 0 { + t.Fatalf("rejected release advertised catalogue: %s", &output) + } + }) + } +} diff --git a/misc/fctl-plugin/internal/catalogue/release.go b/misc/fctl-plugin/internal/catalogue/release.go new file mode 100644 index 0000000000..5e2aeea08d --- /dev/null +++ b/misc/fctl-plugin/internal/catalogue/release.go @@ -0,0 +1,143 @@ +// Package catalogue prepares product release entries for the fctl catalogue. +// Its wire fields follow catalogue schemaVersion 1 without importing the core. +package catalogue + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/url" + "os" + "regexp" + "strings" + + "github.com/formancehq/fctl/pkg/pluginsdk" +) + +type Platform struct { + OS string `json:"os"` + Arch string `json:"arch"` +} + +type Artifact struct { + Registry string `json:"registry"` + Repository string `json:"repository"` + Digest string `json:"digest"` +} + +type Release struct { + Service string `json:"service"` + ServiceVersion string `json:"serviceVersion"` + Revision int `json:"revision"` + Platform Platform `json:"platform"` + Artifact Artifact `json:"artifact"` + SHA256 string `json:"sha256"` + Manifest pluginsdk.Manifest `json:"manifest"` +} + +type Catalogue struct { + SchemaVersion int `json:"schemaVersion"` + Releases []Release `json:"releases"` +} + +const ( + maxManifestBytes = 1 << 20 + maxBinaryBytes = 128 << 20 +) + +var repositoryPattern = regexp.MustCompile(`^[a-z0-9]+(?:[._-][a-z0-9]+)*(?:/[a-z0-9]+(?:[._-][a-z0-9]+)*)*$`) + +// ReadManifest reads the SDK metadata exported by the native executable. +func ReadManifest(path string) (pluginsdk.Manifest, error) { + file, err := os.Open(path) + if err != nil { + return pluginsdk.Manifest{}, err + } + defer func() { _ = file.Close() }() // Read-only resource; read failures are returned below. + data, err := io.ReadAll(io.LimitReader(file, maxManifestBytes+1)) + if err != nil { + return pluginsdk.Manifest{}, err + } + if len(data) > maxManifestBytes { + return pluginsdk.Manifest{}, errors.New("plugin manifest exceeds 1 MiB") + } + var manifest pluginsdk.Manifest + if err := json.Unmarshal(data, &manifest); err != nil { + return pluginsdk.Manifest{}, fmt.Errorf("read SDK manifest: %w", err) + } + + return manifest, nil +} + +// NewRelease binds the actual raw binary checksum to an immutable OCI digest. +// Binary is an explicit operator-selected file, not a path from a manifest. +func NewRelease(manifest pluginsdk.Manifest, binary string, artifact Artifact, platform Platform, revision int) (Release, error) { + if manifest.Name != "ledger" || manifest.Service != "ledger" || pluginsdk.CommandName(manifest.Root) != "ledger" || + manifest.ProtocolVersion != pluginsdk.ProtocolVersion { + return Release{}, errors.New("manifest must describe the Ledger plugin and current SDK protocol") + } + if manifest.Version == "" || len(manifest.Version) > 128 || strings.TrimSpace(manifest.Version) != manifest.Version || + strings.ContainsAny(manifest.Version, "/\\\x00\r\n\t ") || revision < 1 { + return Release{}, errors.New("exact service version and positive plugin revision are required") + } + if platform.OS != "linux" && platform.OS != "darwin" && platform.OS != "windows" || + platform.Arch != "amd64" && platform.Arch != "arm64" { + return Release{}, errors.New("unsupported plugin platform") + } + if err := validateArtifact(artifact); err != nil { + return Release{}, err + } + checksum, err := hashBinary(binary) + if err != nil { + return Release{}, err + } + + return Release{Service: manifest.Service, ServiceVersion: manifest.Version, Revision: revision, + Platform: platform, Artifact: artifact, SHA256: checksum, Manifest: manifest}, nil +} + +func validateArtifact(artifact Artifact) error { + origin, err := url.Parse(artifact.Registry) + if err != nil || origin.Host == "" || origin.User != nil || origin.Path != "" || origin.RawQuery != "" || origin.Fragment != "" { + return errors.New("registry must be an HTTPS origin without credentials") + } + ip := net.ParseIP(origin.Hostname()) + loopback := origin.Hostname() == "localhost" || ip != nil && ip.IsLoopback() + if origin.Scheme != "https" && (origin.Scheme != "http" || !loopback) { + return errors.New("registry requires HTTPS, except loopback development registries") + } + if len(artifact.Repository) > 255 || !repositoryPattern.MatchString(artifact.Repository) { + return errors.New("invalid OCI repository") + } + checksum, ok := strings.CutPrefix(artifact.Digest, "sha256:") + if !ok || len(checksum) != 64 || strings.ToLower(checksum) != checksum { + return errors.New("artifact requires an immutable sha256 digest") + } + if _, err := hex.DecodeString(checksum); err != nil { + return errors.New("artifact requires an immutable sha256 digest") + } + + return nil +} + +func hashBinary(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer func() { _ = file.Close() }() // Read-only resource; all read failures are returned. + checksum := sha256.New() + size, err := io.Copy(checksum, io.LimitReader(file, maxBinaryBytes+1)) + if err != nil { + return "", err + } + if size == 0 || size > maxBinaryBytes { + return "", errors.New("raw plugin binary must contain 1 byte to 128 MiB") + } + + return hex.EncodeToString(checksum.Sum(nil)), nil +} diff --git a/misc/fctl-plugin/internal/catalogue/release_test.go b/misc/fctl-plugin/internal/catalogue/release_test.go new file mode 100644 index 0000000000..aa53b1aba0 --- /dev/null +++ b/misc/fctl-plugin/internal/catalogue/release_test.go @@ -0,0 +1,80 @@ +package catalogue + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + ledger "github.com/formancehq/ledger/misc/fctl-plugin" +) + +func TestReleaseUsesSDKManifestAndActualRawBinaryChecksum(t *testing.T) { + t.Parallel() + manifest, err := ledger.NewVersion(nil, "3.0.0-beta.5").GetManifest(t.Context()) + if err != nil { + t.Fatal(err) + } + directory := t.TempDir() + binary := filepath.Join(directory, "binary") + const data = "synthetic native executable bytes" + if err := os.WriteFile(binary, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + checksum := sha256.Sum256([]byte(data)) + artifact := Artifact{Registry: "https://ghcr.io", Repository: "formancehq/fctl-plugin-ledger", Digest: "sha256:" + strings.Repeat("a", 64)} + release, err := NewRelease(manifest, binary, artifact, Platform{OS: "windows", Arch: "arm64"}, 2) + if err != nil || release.ServiceVersion != "3.0.0-beta.5" || release.Revision != 2 || release.SHA256 != hex.EncodeToString(checksum[:]) { + t.Fatalf("release=%#v, err=%v", release, err) + } + encoded, err := json.Marshal(Catalogue{SchemaVersion: 1, Releases: []Release{release}}) + if err != nil { + t.Fatal(err) + } + var wire map[string]json.RawMessage + if err := json.Unmarshal(encoded, &wire); err != nil || string(wire["schemaVersion"]) != "1" || wire["releases"] == nil { + t.Fatalf("catalogue wire=%s, err=%v", encoded, err) + } + manifestPath := filepath.Join(directory, "manifest.json") + encoded, err = json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(manifestPath, encoded, 0o600); err != nil { + t.Fatal(err) + } + read, err := ReadManifest(manifestPath) + if err != nil || read.Version != manifest.Version || len(read.Root.Subcommands) != len(manifest.Root.Subcommands) { + t.Fatalf("read manifest=%#v, err=%v", read, err) + } +} + +func TestImmutableArtifactValidation(t *testing.T) { + t.Parallel() + valid := Artifact{Registry: "https://ghcr.io", Repository: "formancehq/fctl-plugin-ledger", Digest: "sha256:" + strings.Repeat("a", 64)} + for _, registry := range []string{"https://ghcr.io", "http://localhost:5000", "http://127.0.0.1:5000", "http://[::1]:5000"} { + artifact := valid + artifact.Registry = registry + if err := validateArtifact(artifact); err != nil { + t.Errorf("valid registry %s: %v", registry, err) + } + } + for _, mutate := range []func(*Artifact){ + func(a *Artifact) { a.Digest = "latest" }, + func(a *Artifact) { a.Digest = "sha256:" + strings.Repeat("A", 64) }, + func(a *Artifact) { a.Digest = "sha256:" + strings.Repeat("x", 64) }, + func(a *Artifact) { a.Registry = "http://registry.example" }, + func(a *Artifact) { a.Registry = "https://token@ghcr.io" }, + func(a *Artifact) { a.Registry = "https://ghcr.io/api" }, + func(a *Artifact) { a.Repository = "../secret" }, + } { + artifact := valid + mutate(&artifact) + if err := validateArtifact(artifact); err == nil { + t.Fatalf("accepted invalid artifact: %#v", artifact) + } + } +} diff --git a/misc/fctl-plugin/manifest.go b/misc/fctl-plugin/manifest.go new file mode 100644 index 0000000000..2899ba8a44 --- /dev/null +++ b/misc/fctl-plugin/manifest.go @@ -0,0 +1,174 @@ +// Package ledger implements the Ledger v3 plugin without CLI or configuration dependencies. +// Routes follow release/v3.0 at 8e9bb3ebc13f9235fa20e92fb45018a6f7e91e16. +package ledger + +import ( + "errors" + "net/http" + "strings" + + "github.com/formancehq/fctl/pkg/pluginsdk" + httpclient "github.com/formancehq/fctl/pkg/pluginsdk/httpclient" +) + +type layout struct{} + +type node struct { + spec pluginsdk.CommandSpec + op *operation + children []*node +} + +func group(use, short string) *node { + return &node{spec: pluginsdk.CommandSpec{Use: use, Short: short}} +} + +func (n *node) add(children ...*node) { n.children = append(n.children, children...) } + +func buildLayout() (pluginsdk.Manifest, map[string]operation) { + m := layout{} + root := group("ledger", "Use the Ledger v3 data-plane API") + root.spec.Target = "stack" + root.spec.Long = "Use the Ledger v3 data-plane API. Nested commands select a ledger with --ledger.\nLedger, account, transaction and log lists return one page; continue with --cursor\nusing the response's next or previous token, keeping the same filters and order.\n--after remains a compatibility alias for forward account, transaction and log pagination." + root.spec.Example = "fctl ledger create books\nfctl ledger --ledger books transactions create --data @transaction.json --idempotency-key payment-42" + root.spec.Flags = []pluginsdk.FlagSpec{ + {Name: "ledger", Type: "string", Persistent: true, Usage: "Ledger name for nested commands"}, + {Name: "consistency", Type: "string", Persistent: true, Usage: "Read consistency: linearizable or stale (server default: linearizable)"}, + } + root.add( + m.endpoint(operation{use: "list", short: "List one page of ledgers", global: true, page: true, reverse: true, path: func([]string) []string { return []string{"v3", ""} }}), + m.endpoint(operation{use: "create [name]", short: "Create a ledger", ledgerArg: true, method: http.MethodPost, body: bodyDefault, validateBody: validateLedgerPayload, idempotency: true}), + m.endpoint(operation{use: "show [name]", short: "Show a ledger", ledgerArg: true}), + m.endpoint(operation{use: "delete [name]", short: "Delete a ledger", ledgerArg: true, method: http.MethodDelete, idempotency: true}), + m.endpoint(operation{use: "stats", short: "Show ledger statistics", path: fixed("stats")}), + m.endpoint(operation{use: "balances", short: "Aggregate ledger balances and volumes", path: fixed("volumes"), filter: true, + boolQuery: map[string]string{"use-max-precision": "useMaxPrecision", "collapse-colors": "collapseColors"}, stringQuery: map[string]string{"group-by-prefixes": "groupByPrefixes"}}), + m.endpoint(operation{use: "info", short: "Show Ledger server version information", global: true, path: func([]string) []string { return []string{"_info"} }}), + m.accounts(), m.transactions(), m.metadata(nil, false), m.indexes(), + ) + logs := group("logs", "Read ledger logs (requires the LOG index)") + logs.add(m.endpoint(operation{use: "list", short: "List one page of ledger logs", path: fixed("logs"), page: true, reverse: true, afterID: true, filter: true, dates: true})) + root.add(logs) + bulk := m.endpoint(operation{use: "bulk", short: "Submit v3 bulk operations once", method: http.MethodPost, path: fixed("bulk"), body: bodyRequired, validateBody: validateBulkPayload, idempotency: true, bulk: true, + boolQuery: map[string]string{"atomic": "atomic", "continue-on-failure": "continueOnFailure"}}) + bulk.spec.Long = "Submit a JSON array of v3 bulk operations once. With --atomic, --idempotency-key identifies the whole batch. Otherwise each element's ik identifies its operation and the header is ignored. Inspect every returned element for business failures, including when --continue-on-failure is enabled. This is not a log import or backup restore." + root.add(bulk) + operations := make(map[string]operation) + spec := publish(root, nil, operations) + + return pluginsdk.Manifest{Name: "ledger", Version: "3.0.0", Service: "ledger", ProtocolVersion: pluginsdk.ProtocolVersion, Root: spec}, operations +} + +func (m layout) accounts() *node { + group := group("accounts", "Read accounts and balances, update metadata") + group.add( + m.endpoint(operation{use: "list", short: "List accounts", path: fixed("accounts"), page: true, reverse: true, filter: true}), + m.endpoint(operation{use: "show
", short: "Show an account including volumes and metadata", args: 1, path: resource("accounts"), boolQuery: map[string]string{"collapse-colors": "collapseColors"}}), + m.endpoint(operation{use: "balances
", short: "Show account balances in the full account envelope", args: 1, path: resource("accounts"), boolQuery: map[string]string{"collapse-colors": "collapseColors"}}), + m.metadata([]string{"accounts"}, false), + ) + + return group +} + +func (m layout) transactions() *node { + group := group("transactions", "Read, create and revert transactions") + group.add( + m.endpoint(operation{use: "list", short: "List transactions (newest first by default)", path: fixed("transactions"), page: true, afterID: true, reverse: true, filter: true, dates: true}), + m.endpoint(operation{use: "show ", short: "Show a transaction", args: 1, transactionID: true, path: resource("transactions")}), + m.endpoint(operation{use: "create", short: "Create a transaction from v3 JSON (postings or Numscript)", method: http.MethodPost, path: fixed("transactions"), body: bodyRequired, validateBody: validateTransactionPayload, idempotency: true}), + m.endpoint(operation{use: "revert ", short: "Revert a transaction", args: 1, transactionID: true, method: http.MethodPost, path: resource("transactions", "revert"), body: bodyOptional, validateBody: validateRevertPayload, idempotency: true}), + m.metadata([]string{"transactions"}, true), + ) + + return group +} + +// Metadata reads use the owning resource's GET route: v3 has no GET /metadata. +func (m layout) metadata(parent []string, transactionID bool) *node { + group := group("metadata", "Read metadata in the resource envelope, set or delete keys") + args := 0 + argLabel := "" + if len(parent) > 0 { + args = 1 + argLabel = "
" + if transactionID { + argLabel = " " + } + } + base := func(a []string) []string { + segments := append([]string{}, parent...) + if args > 0 { + segments = append(segments, a[0]) + } + + return segments + } + group.add( + m.endpoint(operation{use: "show" + argLabel, short: "Show the resource including metadata", args: args, transactionID: transactionID, path: base}), + m.endpoint(operation{use: "set" + argLabel, short: "Merge a JSON metadata object", args: args, transactionID: transactionID, method: http.MethodPost, body: bodyRequired, validateBody: validateMetadataPayload, idempotency: true, path: func(a []string) []string { return append(base(a), "metadata") }}), + m.endpoint(operation{use: "delete" + argLabel + " ", short: "Delete one raw metadata key", args: args + 1, transactionID: transactionID, method: http.MethodDelete, idempotency: true, path: func(a []string) []string { return append(base(a), "metadata", a[args]) }}), + ) + + return group +} + +func (m layout) indexes() *node { + group := group("indexes", "Manage and inspect ledger indexes") + group.add( + m.endpoint(operation{use: "list", short: "List ledger indexes", path: fixed("indexes")}), + m.endpoint(operation{use: "show ", short: "Show an index", args: 1, path: resource("indexes")}), + m.endpoint(operation{use: "status ", short: "Show index build status", args: 1, path: resource("indexes", "status")}), + m.endpoint(operation{use: "create", short: "Create an index from JSON, e.g. {\"id\":\"log_builtin:LOG_BUILTIN_INDEX_DATE\"}", method: http.MethodPost, path: fixed("indexes"), body: bodyRequired, validateBody: validateIndexPayload, idempotency: true}), + m.endpoint(operation{use: "delete ", short: "Drop an index", args: 1, method: http.MethodDelete, path: resource("indexes"), idempotency: true}), + ) + inspect := m.endpoint(operation{use: "inspect ", short: "Inspect a metadata index", args: 1, path: resource("indexes", "inspect"), inspect: true}) + group.add(inspect) + + return group +} + +func fixed(segments ...string) func([]string) []string { + return func([]string) []string { return segments } +} + +func resource(collection string, suffix ...string) func([]string) []string { + return func(args []string) []string { return append([]string{collection, args[0]}, suffix...) } +} + +func requestPath(op operation, args []string, selected string) (string, error) { + if op.global { + return httpclient.Path(op.path(args)...), nil + } + name := selected + if op.ledgerArg && len(args) > 0 { + if name != "" && name != args[0] { + return "", errors.New("positional ledger name conflicts with --ledger") + } + name = args[0] + } + if name == "" { + return "", errors.New("select a ledger with --ledger (create/show/delete also accept a positional name)") + } + segments := []string{"v3", name} + if op.path != nil { + segments = append(segments, op.path(args)...) + } + + return httpclient.Path(segments...), nil +} + +func publish(n *node, parent []string, operations map[string]operation) pluginsdk.CommandSpec { + name := strings.Fields(n.spec.Use)[0] + path := append(append([]string{}, parent...), name) + spec := n.spec + if n.op != nil { + operations[strings.Join(path, "/")] = *n.op + spec.Inputs = append(spec.Inputs, operationInputs(path, *n.op)...) + } + for _, child := range n.children { + spec.Subcommands = append(spec.Subcommands, publish(child, path, operations)) + } + + return spec +} diff --git a/misc/fctl-plugin/pagination_test.go b/misc/fctl-plugin/pagination_test.go new file mode 100644 index 0000000000..7eda135a12 --- /dev/null +++ b/misc/fctl-plugin/pagination_test.go @@ -0,0 +1,165 @@ +package ledger + +import ( + "context" + "encoding/base64" + "maps" + "reflect" + "strings" + "testing" + + "github.com/formancehq/fctl/pkg/pluginsdk" +) + +func TestUnsupportedPaginationRejectedBeforeRequest(t *testing.T) { + t.Parallel() + for _, args := range [][]string{ + {"list", "--after", "books"}, + {"--ledger", "books", "indexes", "inspect", "metadata:TARGET_TYPE_ACCOUNT:key", "--after", "42"}, + } { + out, requests, err := execute(t, args, "", envelope, 200) + if err == nil || len(requests) != 0 || out != "" { + t.Fatalf("unsupported flags accepted: %v; requests=%v out=%s err=%v", args, requests, out, err) + } + } +} + +func TestAfterIDValidationBeforeRequest(t *testing.T) { + t.Parallel() + for _, collection := range []string{"transactions", "logs"} { + for _, after := range []string{"opaque/+=", "-1", "18446744073709551616"} { + out, requests, err := execute(t, []string{"--ledger", "books", collection, "list", "--after", after}, "", envelope, 200) + if err == nil || len(requests) != 0 || out != "" { + t.Fatalf("invalid %s ID %q reached server: requests=%v err=%v", collection, after, requests, err) + } + } + } +} + +func TestLedgerListReturnsOnePageWithDefaultSize(t *testing.T) { + t.Parallel() + response := `{"data":[{"name":"books"}],"next":"next-page","hasMore":true}` + out, requests, err := execute(t, []string{"list"}, "", response, 200) + if err != nil || out != response || len(requests) != 1 { + t.Fatalf("list page: out=%s requests=%v err=%v", out, requests, err) + } + if requests[0].query.Get("pageSize") != "100" || len(requests[0].query) != 1 || requests[0].path != "/gateway/ledger/v3/" { + t.Fatalf("list pagination: %+v", requests[0]) + } +} + +func TestCursorQueryAndAfterCompatibility(t *testing.T) { + t.Parallel() + for _, collection := range []string{"", "accounts", "transactions", "logs"} { + for _, backward := range []bool{false, true} { + t.Run(collection+map[bool]string{false: "/next", true: "/previous"}[backward], func(t *testing.T) { + t.Parallel() + key := "18446744073709551615" + if collection == "accounts" || collection == "" { + key = "users:next/+=" + } + cursorJSON := `{"key":"` + key + `"}` + if backward { + cursorJSON = `{"key":"` + key + `","back":true}` + } + token := base64.RawURLEncoding.EncodeToString([]byte(cursorJSON)) + args := []string{"--consistency", "stale"} + if collection != "" { + args = append(args, "--ledger", "books", collection) + } + args = append(args, "list", "--page-size", "0", "--reverse=false", "--cursor", token) + _, requests, err := execute(t, args, "", envelope, 200) + if err != nil || len(requests) != 1 { + t.Fatalf("cursor query: requests=%v err=%v", requests, err) + } + checkQuery(t, requests[0].query, map[string]string{"cursor": token, "pageSize": "0", "reverse": "false"}) + if requests[0].query.Has("after") || requests[0].header.Get("X-Consistency") != "stale" { + t.Fatalf("cursor or consistency lost: %+v", requests[0]) + } + if collection == "" || backward { + return + } + args = append(args[:len(args)-2], "--after", key) + _, requests, err = execute(t, args, "", envelope, 200) + if err != nil || len(requests) != 1 || requests[0].query.Get("cursor") != token || requests[0].query.Has("after") { + t.Fatalf("after alias did not use canonical cursor: requests=%v err=%v", requests, err) + } + }) + } + } +} + +func TestPaginationNormalizationForAlternateExecutor(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + collection, after, key string + changed map[string]bool + }{ + {"accounts", "users:42/+=", "users:42/+=", map[string]bool{"after": true}}, + {"transactions", "00042", "42", map[string]bool{"after": true, "reverse": true}}, + {"logs", "18446744073709551615", "18446744073709551615", nil}, + } { + t.Run(tc.collection, func(t *testing.T) { + t.Parallel() + token := base64.RawURLEncoding.EncodeToString([]byte(`{"key":"` + tc.key + `"}`)) + calls := 0 + p := NewWithExecutor("", func(_ context.Context, req pluginsdk.ExecuteRequest) (pluginsdk.ExecuteResponse, error) { + calls++ + _, afterFlag := req.Flags["after"] + _, afterChanged := req.ChangedFlags["after"] + if req.Flags["cursor"] != token || !req.ChangedFlags["cursor"] || afterFlag || afterChanged { + t.Fatalf("executor received an unnormalized alias: %#v", req) + } + if req.ChangedFlags["reverse"] != tc.changed["reverse"] { + t.Fatalf("pagination normalization lost another changed flag: %#v", req) + } + + return pluginsdk.ExecuteResponse{}, nil + }) + req := pluginsdk.ExecuteRequest{CommandPath: []string{"ledger", tc.collection, "list"}, Flags: map[string]string{"ledger": "books", "after": tc.after}, ChangedFlags: tc.changed} + beforeFlags, beforeChanged := maps.Clone(req.Flags), maps.Clone(req.ChangedFlags) + if _, err := p.Execute(t.Context(), req); err != nil || calls != 1 { + t.Fatalf("normalized execute: calls=%d err=%v", calls, err) + } + if !reflect.DeepEqual(req.Flags, beforeFlags) || !reflect.DeepEqual(req.ChangedFlags, beforeChanged) { + t.Fatal("pagination normalization mutated caller maps") + } + }) + } +} + +func TestInvalidCursorRejectedBeforeExecutors(t *testing.T) { + t.Parallel() + for _, raw := range []string{`null`, `[]`, `{"key":null}`, `{"key":42}`, `{"key":"-1"}`, `{"key":"18446744073709551616"}`, `{"back":"true"}`, `{"back":null}`, `{"other":true}`, `{} {}`} { + t.Run(raw, func(t *testing.T) { + t.Parallel() + token := base64.RawURLEncoding.EncodeToString([]byte(raw)) + assertPaginationRejected(t, []string{"--ledger", "books", "transactions", "list", "--cursor", token}, "invalid --cursor") + }) + } + assertPaginationRejected(t, []string{"--ledger", "books", "transactions", "list", "--cursor", "not/base64"}, "invalid --cursor") + for _, collection := range []string{"accounts", "transactions", "logs"} { + assertPaginationRejected(t, []string{"--ledger", "books", collection, "list", "--cursor", "eyJrZXkiOiI0MiJ9", "--after", "42"}, "cannot be used together") + } +} + +func assertPaginationRejected(t *testing.T, tokens []string, want string) { + t.Helper() + out, requests, err := execute(t, tokens, "", envelope, 200) + if err == nil || !strings.Contains(err.Error(), want) || len(requests) != 0 || out != "" { + t.Fatalf("HTTP: want %q before dispatch; requests=%v out=%s err=%v", want, requests, out, err) + } + calls := 0 + p := NewWithExecutor("", func(context.Context, pluginsdk.ExecuteRequest) (pluginsdk.ExecuteResponse, error) { + calls++ + + return pluginsdk.ExecuteResponse{}, nil + }) + manifest, err := p.GetManifest(t.Context()) + if err != nil { + t.Fatal(err) + } + if _, err := p.Execute(t.Context(), testRequest(manifest, tokens, "")); err == nil || !strings.Contains(err.Error(), want) || calls != 0 { + t.Fatalf("alternate executor: want %q before dispatch; calls=%d err=%v", want, calls, err) + } +} diff --git a/misc/fctl-plugin/payload.go b/misc/fctl-plugin/payload.go new file mode 100644 index 0000000000..7e28102b21 --- /dev/null +++ b/misc/fctl-plugin/payload.go @@ -0,0 +1,359 @@ +package ledger + +import ( + "bytes" + "encoding/json" + "fmt" + "math/big" + "strconv" + "strings" +) + +type payloadValidator func(json.RawMessage, string) error + +// These validators check the public HTTP payload shape before either executor. +// Missing fields and null optional fields retain the server's defaults. Unknown +// fields are left untouched for future contracts; schema, account existence, +// balance checks and other state-dependent rules remain server responsibilities. +// Validation never remarshals the body or decodes a number through float64. +func validateTransactionPayload(raw json.RawMessage, path string) error { + return payloadFields(raw, path, map[string]payloadValidator{ + "postings": payloadArray(validatePostingPayload), + "script": validateScriptPayload, + "scriptReference": validateScriptReferencePayload, + "timestamp": validateJSONString, + "reference": validateJSONString, + "metadata": validateMetadataPayload, + "accountMetadata": payloadMap(validateMetadataPayload), + "force": validateJSONBool, + }) +} + +func validatePostingPayload(raw json.RawMessage, path string) error { + return payloadFields(raw, path, map[string]payloadValidator{ + "source": validateJSONString, "destination": validateJSONString, + "asset": validateJSONString, "color": validateJSONString, + "amount": validatePostingAmount, + }) +} + +func validateScriptPayload(raw json.RawMessage, path string) error { + return payloadFields(raw, path, map[string]payloadValidator{ + "plain": validateJSONString, "vars": payloadMap(validateJSONString), + }) +} + +func validateScriptReferencePayload(raw json.RawMessage, path string) error { + return payloadFields(raw, path, map[string]payloadValidator{ + "name": validateJSONString, "version": validateJSONString, + "vars": payloadMap(validateJSONString), + }) +} + +func validateRevertPayload(raw json.RawMessage, path string) error { + return payloadFields(raw, path, map[string]payloadValidator{ + "force": validateJSONBool, "atEffectiveDate": validateJSONBool, + "metadata": validateMetadataPayload, + }) +} + +func validateLedgerPayload(raw json.RawMessage, path string) error { + return payloadFields(raw, path, map[string]payloadValidator{ + "metadata": validateMetadataPayload, "mode": validateJSONString, + "defaultEnforcementMode": validateJSONString, + "mirrorSource": validateMirrorPayload, + "initialSchema": payloadArray(func(raw json.RawMessage, path string) error { + return payloadFields(raw, path, map[string]payloadValidator{ + "targetType": validateJSONString, "key": validateJSONString, "type": validateJSONString, + }) + }), + "accountTypes": payloadMap(validateAccountTypePayload), + }) +} + +func validateMirrorPayload(raw json.RawMessage, path string) error { + return payloadFields(raw, path, map[string]payloadValidator{ + "ledgerName": validateJSONString, "type": validateJSONString, + "baseUrl": validateJSONString, "dsn": validateJSONString, + "oauth2ClientId": validateJSONString, "oauth2ClientSecret": validateJSONString, + "oauth2TokenEndpoint": validateJSONString, + "oauth2Scopes": payloadArray(validateJSONString), + "batchSize": validateJSONUint32, + "rewriteRules": payloadArray(func(raw json.RawMessage, path string) error { + _, err := payloadObject(raw, path) + + return err + }), + }) +} + +func validateAccountTypePayload(raw json.RawMessage, path string) error { + return payloadFields(raw, path, map[string]payloadValidator{ + "name": validateJSONString, "pattern": validateJSONString, "persistence": validateJSONString, + "segmentTypes": payloadMap(func(raw json.RawMessage, path string) error { + return payloadFields(raw, path, map[string]payloadValidator{ + "type": validateJSONString, "regex": validateJSONString, + }) + }), + }) +} + +func validateIndexPayload(raw json.RawMessage, path string) error { + return payloadFields(raw, path, map[string]payloadValidator{"id": validateJSONString}) +} + +func validateMetadataPayload(raw json.RawMessage, path string) error { + return payloadMap(func(raw json.RawMessage, path string) error { + var value any + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + if err := decoder.Decode(&value); err != nil { + return fmt.Errorf("%s: %w", path, err) + } + switch value := value.(type) { + case nil, string, bool: + return nil + case json.Number: + return validateMetadataNumber(value.String(), path) + default: + return fmt.Errorf("%s must be a string, boolean, exact integer or null", path) + } + })(raw, path) +} + +func validateBulkPayload(raw json.RawMessage, path string) error { + return payloadArray(func(raw json.RawMessage, path string) error { + fields, err := payloadObject(raw, path) + if err != nil { + return err + } + var action string + if err := validateJSONString(fields["action"], path+".action"); err != nil { + return err + } + if err := json.Unmarshal(fields["action"], &action); err != nil { + return fmt.Errorf("%s.action: %w", path, err) + } + if action == "" { + return fmt.Errorf("%s.action must not be empty", path) + } + if err := payloadFields(raw, path, map[string]payloadValidator{ + "ik": validateIdempotencyPayload, "skippableReasons": payloadArray(validateJSONString), + }); err != nil { + return err + } + dataPath := path + ".data" + data, err := payloadObject(fields["data"], dataPath) + if err != nil { + return err + } + switch action { + case "CREATE_TRANSACTION": + return validateTransactionPayload(fields["data"], dataPath) + case "REVERT_TRANSACTION": + if err := validateRevertPayload(fields["data"], dataPath); err != nil { + return err + } + if id, ok := data["id"]; ok { + return validateJSONUint64(id, dataPath+".id") + } + case "ADD_METADATA", "DELETE_METADATA": + if err := payloadFields(fields["data"], dataPath, map[string]payloadValidator{ + "targetType": validateJSONString, "metadata": validateMetadataPayload, "key": validateJSONString, + }); err != nil { + return err + } + if target, ok := data["targetId"]; ok { + var kind string + if err := json.Unmarshal(data["targetType"], &kind); err != nil { + return fmt.Errorf("%s.targetType must be a string", dataPath) + } + switch strings.ToUpper(kind) { + case "ACCOUNT": + return validateJSONString(target, dataPath+".targetId") + case "TRANSACTION": + return validateJSONUint64(target, dataPath+".targetId") + } + } + } + // Future actions keep their object payload and are validated by the server. + return nil + })(raw, path) +} + +func payloadObject(raw json.RawMessage, path string) (map[string]json.RawMessage, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil || fields == nil { + return nil, fmt.Errorf("%s must be a JSON object", path) + } + + return fields, nil +} + +func payloadFields(raw json.RawMessage, path string, validators map[string]payloadValidator) error { + fields, err := payloadObject(raw, path) + if err != nil { + return err + } + for key, validate := range validators { + value, ok := fields[key] + if !ok || bytes.Equal(bytes.TrimSpace(value), []byte("null")) { + continue + } + if err := validate(value, path+"."+key); err != nil { + return err + } + } + + return nil +} + +func payloadMap(validate payloadValidator) payloadValidator { + return func(raw json.RawMessage, path string) error { + fields, err := payloadObject(raw, path) + if err != nil { + return err + } + for key, value := range fields { + if err := validate(value, fmt.Sprintf("%s[%q]", path, key)); err != nil { + return err + } + } + + return nil + } +} + +func payloadArray(validate payloadValidator) payloadValidator { + return func(raw json.RawMessage, path string) error { + var values []json.RawMessage + if err := json.Unmarshal(raw, &values); err != nil || values == nil { + return fmt.Errorf("%s must be a JSON array", path) + } + for i, value := range values { + if err := validate(value, fmt.Sprintf("%s[%d]", path, i)); err != nil { + return err + } + } + + return nil + } +} + +func validateJSONString(raw json.RawMessage, path string) error { + var value string + if err := json.Unmarshal(raw, &value); err != nil || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return fmt.Errorf("%s must be a string", path) + } + + return nil +} + +func validateJSONBool(raw json.RawMessage, path string) error { + value := string(bytes.TrimSpace(raw)) + if value != "true" && value != "false" { + return fmt.Errorf("%s must be a boolean", path) + } + + return nil +} + +func validateJSONUint32(raw json.RawMessage, path string) error { + if _, err := strconv.ParseUint(string(bytes.TrimSpace(raw)), 10, 32); err != nil { + return fmt.Errorf("%s must be an unsigned 32-bit integer", path) + } + + return nil +} + +func validateJSONUint64(raw json.RawMessage, path string) error { + if _, err := strconv.ParseUint(string(bytes.TrimSpace(raw)), 10, 64); err != nil { + return fmt.Errorf("%s must be an unsigned 64-bit integer", path) + } + + return nil +} + +func validatePostingAmount(raw json.RawMessage, path string) error { + raw = bytes.TrimSpace(raw) + decimal := string(raw) + if len(raw) > 0 && raw[0] == '"' { + if err := json.Unmarshal(raw, &decimal); err != nil { + return fmt.Errorf("%s must be an unsigned decimal integer", path) + } + } + if decimal == "" || len(decimal) > 78 || (len(decimal) > 1 && decimal[0] == '0') { + return fmt.Errorf("%s must be a canonical unsigned decimal integer within uint256", path) + } + for _, digit := range decimal { + if digit < '0' || digit > '9' { + return fmt.Errorf("%s must be a canonical unsigned decimal integer within uint256", path) + } + } + value, ok := new(big.Int).SetString(decimal, 10) + if !ok || value.BitLen() > 256 { + return fmt.Errorf("%s exceeds uint256", path) + } + + return nil +} + +func validateIdempotencyPayload(raw json.RawMessage, path string) error { + if err := validateJSONString(raw, path); err != nil { + return err + } + var key string + if err := json.Unmarshal(raw, &key); err != nil { + return fmt.Errorf("%s: %w", path, err) + } + if len(key) > 256 || strings.ContainsAny(key, "\r\n") { + return fmt.Errorf("%s must be at most 256 bytes without newlines", path) + } + + return nil +} + +// Integral decimal/exponent forms are valid metadata. Bound exponent work by +// token length and the 20-digit uint64 limit, never by an untrusted exponent. +func validateMetadataNumber(number, path string) error { + failure := func() error { return fmt.Errorf("%s must be an exact integer within int64/uint64", path) } + negative := strings.HasPrefix(number, "-") + mantissa := strings.TrimPrefix(number, "-") + exponentText := "0" + if i := strings.IndexAny(mantissa, "eE"); i >= 0 { + exponentText, mantissa = mantissa[i+1:], mantissa[:i] + } + fraction := 0 + if i := strings.IndexByte(mantissa, '.'); i >= 0 { + fraction = len(mantissa) - i - 1 + mantissa = mantissa[:i] + mantissa[i+1:] + } + digits := strings.TrimLeft(mantissa, "0") + if digits == "" { + return nil + } + exponent, err := strconv.ParseInt(exponentText, 10, 64) + limit := int64(len(number)) + 20 + if err != nil || exponent < -limit || exponent > limit { + return failure() + } + shift := exponent - int64(fraction) + if shift < 0 { + zeros := len(digits) - len(strings.TrimRight(digits, "0")) + if -shift > int64(zeros) { + return failure() + } + digits = digits[:len(digits)+int(shift)] + } else { + if int64(len(digits))+shift > 20 { + return failure() + } + digits += strings.Repeat("0", int(shift)) + } + value, err := strconv.ParseUint(digits, 10, 64) + if err != nil || (negative && value > uint64(1)<<63) { + return failure() + } + + return nil +} diff --git a/misc/fctl-plugin/payload_test.go b/misc/fctl-plugin/payload_test.go new file mode 100644 index 0000000000..7bb9794625 --- /dev/null +++ b/misc/fctl-plugin/payload_test.go @@ -0,0 +1,135 @@ +package ledger + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/formancehq/fctl/pkg/pluginsdk" +) + +func TestPayloadValidationBeforeExecutors(t *testing.T) { + t.Parallel() + cases := []struct{ name, command, body, want string }{ + {"ledger array", "create books", `[]`, "body must be a JSON object"}, + {"ledger mode", "create books", `{"mode":1}`, "body.mode must be a string"}, + {"schema shape", "create books", `{"initialSchema":{}}`, "body.initialSchema must be a JSON array"}, + {"schema entry", "create books", `{"initialSchema":[false]}`, "body.initialSchema[0] must be a JSON object"}, + {"schema type", "create books", `{"initialSchema":[{"targetType":1}]}`, "body.initialSchema[0].targetType must be a string"}, + {"account type", "create books", `{"accountTypes":{"user":{"pattern":1}}}`, `body.accountTypes["user"].pattern must be a string`}, + {"segment type", "create books", `{"accountTypes":{"user":{"segmentTypes":{"id":[]}}}}`, `body.accountTypes["user"].segmentTypes["id"] must be a JSON object`}, + {"mirror shape", "create books", `{"mirrorSource":[]}`, "body.mirrorSource must be a JSON object"}, + {"mirror batch", "create books", `{"mirrorSource":{"batchSize":4294967296}}`, "body.mirrorSource.batchSize must be an unsigned 32-bit integer"}, + {"mirror scopes", "create books", `{"mirrorSource":{"oauth2Scopes":[false]}}`, "body.mirrorSource.oauth2Scopes[0] must be a string"}, + {"transaction null", "transactions create", `null`, "body must be a JSON object"}, + {"postings shape", "transactions create", `{"postings":{}}`, "body.postings must be a JSON array"}, + {"posting null", "transactions create", `{"postings":[null]}`, "body.postings[0] must be a JSON object"}, + {"posting address", "transactions create", `{"postings":[{"source":12}]}`, "body.postings[0].source must be a string"}, + {"amount float", "transactions create", `{"postings":[{"amount":1.5}]}`, "body.postings[0].amount must be a canonical unsigned decimal integer"}, + {"amount exponent", "transactions create", `{"postings":[{"amount":1e2}]}`, "body.postings[0].amount must be a canonical unsigned decimal integer"}, + {"amount negative", "transactions create", `{"postings":[{"amount":-1}]}`, "body.postings[0].amount must be a canonical unsigned decimal integer"}, + {"amount noncanonical", "transactions create", `{"postings":[{"amount":"01"}]}`, "body.postings[0].amount must be a canonical unsigned decimal integer"}, + {"amount overflow", "transactions create", `{"postings":[{"amount":115792089237316195423570985008687907853269984665640564039457584007913129639936}]}`, "body.postings[0].amount exceeds uint256"}, + {"script shape", "transactions create", `{"script":"send"}`, "body.script must be a JSON object"}, + {"script variable", "transactions create", `{"script":{"vars":{"amount":9007199254740993}}}`, `body.script.vars["amount"] must be a string`}, + {"script reference", "transactions create", `{"scriptReference":{"version":1}}`, "body.scriptReference.version must be a string"}, + {"timestamp", "transactions create", `{"timestamp":123}`, "body.timestamp must be a string"}, + {"force", "transactions create", `{"force":"true"}`, "body.force must be a boolean"}, + {"metadata nested", "metadata set", `{"nested":{}}`, `body["nested"] must be a string, boolean, exact integer or null`}, + {"metadata array", "accounts metadata set users:42", `[]`, "body must be a JSON object"}, + {"metadata fractional", "transactions metadata set 42", `{"score":0.5}`, `body["score"] must be an exact integer`}, + {"metadata positive overflow", "metadata set", `{"score":18446744073709551616}`, `body["score"] must be an exact integer`}, + {"metadata negative overflow", "metadata set", `{"score":-9223372036854775809}`, `body["score"] must be an exact integer`}, + {"metadata exponent overflow", "metadata set", `{"score":1e999999999999999999999999}`, `body["score"] must be an exact integer`}, + {"account metadata", "transactions create", `{"accountMetadata":{"users:42":[]}}`, `body.accountMetadata["users:42"] must be a JSON object`}, + {"revert option", "transactions revert 42", `{"atEffectiveDate":[]}`, "body.atEffectiveDate must be a boolean"}, + {"index id", "indexes create", `{"id":{}}`, "body.id must be a string"}, + {"bulk shape", "bulk", `{}`, "body must be a JSON array"}, + {"bulk entry", "bulk", `[null]`, "body[0] must be a JSON object"}, + {"bulk action", "bulk", `[{"action":false,"data":{}}]`, "body[0].action must be a string"}, + {"bulk missing action", "bulk", `[{"data":{}}]`, "body[0].action must be a string"}, + {"bulk empty action", "bulk", `[{"action":"","data":{}}]`, "body[0].action must not be empty"}, + {"bulk missing data", "bulk", `[{"action":"CREATE_TRANSACTION"}]`, "body[0].data must be a JSON object"}, + {"bulk identity", "bulk", `[{"action":"CREATE_TRANSACTION","data":{},"ik":"bad\nkey"}]`, "body[0].ik must be at most 256 bytes without newlines"}, + {"bulk data", "bulk", `[{"action":"CREATE_TRANSACTION","data":[]}]`, "body[0].data must be a JSON object"}, + {"bulk transaction", "bulk", `[{"action":"CREATE_TRANSACTION","data":{"force":1}}]`, "body[0].data.force must be a boolean"}, + {"bulk revert id", "bulk", `[{"action":"REVERT_TRANSACTION","data":{"id":18446744073709551616}}]`, "body[0].data.id must be an unsigned 64-bit integer"}, + {"bulk account target", "bulk", `[{"action":"ADD_METADATA","data":{"targetType":"ACCOUNT","targetId":1}}]`, "body[0].data.targetId must be a string"}, + {"bulk transaction target", "bulk", `[{"action":"DELETE_METADATA","data":{"targetType":"TRANSACTION","targetId":-1}}]`, "body[0].data.targetId must be an unsigned 64-bit integer"}, + {"bulk lowercase target", "bulk", `[{"action":"DELETE_METADATA","data":{"targetType":"transaction","targetId":"42"}}]`, "body[0].data.targetId must be an unsigned 64-bit integer"}, + {"bulk reasons", "bulk", `[{"action":"CREATE_TRANSACTION","data":{},"skippableReasons":[1]}]`, "body[0].skippableReasons[0] must be a string"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tokens := append([]string{"--ledger", "books"}, strings.Fields(tc.command)...) + out, requests, err := execute(t, tokens, tc.body, envelope, 200) + if err == nil || !strings.Contains(err.Error(), tc.want) || len(requests) != 0 || out != "" { + t.Fatalf("HTTP: want %q before dispatch; requests=%d out=%s err=%v", tc.want, len(requests), out, err) + } + calls := 0 + p := NewWithExecutor("", func(context.Context, pluginsdk.ExecuteRequest) (pluginsdk.ExecuteResponse, error) { + calls++ + + return pluginsdk.ExecuteResponse{}, nil + }) + manifest, err := p.GetManifest(t.Context()) + if err != nil { + t.Fatal(err) + } + _, err = p.Execute(t.Context(), testRequest(manifest, tokens, tc.body)) + if err == nil || !strings.Contains(err.Error(), tc.want) || calls != 0 { + t.Fatalf("alternate executor: want %q before dispatch; calls=%d err=%v", tc.want, calls, err) + } + }) + } +} + +func TestPayloadPreservesCanonicalJSONAndServerValidation(t *testing.T) { + t.Parallel() + cases := []struct{ command, body string }{ + {"create books", `{}`}, {"transactions create", `{}`}, + {"transactions create", `{"postings":[],"metadata":null,"script":null}`}, + {"transactions create", `{"postings":[{"amount":null}],"force":null}`}, + {"transactions revert 42", `{}`}, {"indexes create", `{}`}, + {"metadata set", `{}`}, {"bulk", `[]`}, + {"create books", `{"metadata":{"big":18446744073709551615,"negative":-9223372036854775808},"initialSchema":[{"targetType":"account","key":"id","type":"uint64"}],"accountTypes":{"user":{"name":"user","pattern":"users:{id}","persistence":"EPHEMERAL","segmentTypes":{"id":{"type":"uint64"}}}},"future":{"nested":[1.5]}}`}, + {"create books", `{"mode":"MIRROR","mirrorSource":{"type":"http","baseUrl":"https://example.com","oauth2Scopes":["ledger:read"],"batchSize":4294967295,"rewriteRules":[{"anyVariant":{"actions":[]}}]}}`}, + {"transactions create", ` {"postings":[{"source":"world","destination":"users:42","asset":"USD/2","amount":115792089237316195423570985008687907853269984665640564039457584007913129639935,"color":"RED","future":[]},{"amount":"9007199254740993"}],"metadata":{"exact":9007199254740993},"accountMetadata":{"users:42":{"active":true}},"future":[9007199254740993]} `}, + {"transactions create", `{"script":{"plain":"send [USD/2 $amount]","vars":{"amount":"9007199254740993"}},"timestamp":"2026-10-09T12:00:00Z","reference":"payment","force":false}`}, + {"transactions create", `{"scriptReference":{"name":"pay","version":"latest","vars":{"amount":"9007199254740993"}}}`}, + {"metadata set", `{"integral":9007199254740993.0,"exponent":90071992547409930e-1,"max":1.8446744073709551615e19,"zero":0e99999999999999999999999,"deleted":null,"enabled":true}`}, + {"metadata set", `{"negative":-9223372036854775808.0,"one":100000000000000000000e-20,"small":1e-0,"negativeZero":-0.0}`}, + {"transactions revert 42", `{"force":true,"atEffectiveDate":false,"metadata":{"score":9007199254740993}}`}, + {"indexes create", `{"id":"metadata:TARGET_TYPE_ACCOUNT:key","future":{"option":true}}`}, + {"bulk", `[{"action":"CREATE_TRANSACTION","ik":"tx","data":{}},{"action":"REVERT_TRANSACTION","data":{"id":18446744073709551615}},{"action":"ADD_METADATA","data":{"targetType":"ACCOUNT","targetId":"users:42","metadata":{"exact":9007199254740993}}},{"action":"DELETE_METADATA","data":{"targetType":"TRANSACTION","targetId":42,"key":"key"},"skippableReasons":["METADATA_NOT_FOUND"]},{"action":"FUTURE_ACTION","data":{"future":[1.5]}}]`}, + } + for _, tc := range cases { + t.Run(tc.command+"/"+tc.body, func(t *testing.T) { + t.Parallel() + tokens := append([]string{"--ledger", "books"}, strings.Fields(tc.command)...) + response := `{"data":[]}` + _, requests, err := execute(t, tokens, tc.body, response, 200) + if err != nil || len(requests) != 1 || requests[0].body != tc.body { + t.Fatalf("HTTP changed valid payload: requests=%v err=%v", requests, err) + } + calls := 0 + p := NewWithExecutor("", func(_ context.Context, req pluginsdk.ExecuteRequest) (pluginsdk.ExecuteResponse, error) { + calls++ + if string(req.Body) != tc.body { + t.Fatalf("alternate executor changed exact JSON: %s", req.Body) + } + + return pluginsdk.ExecuteResponse{Data: json.RawMessage(response)}, nil + }) + manifest, err := p.GetManifest(t.Context()) + if err != nil { + t.Fatal(err) + } + if _, err := p.Execute(t.Context(), testRequest(manifest, tokens, tc.body)); err != nil || calls != 1 { + t.Fatalf("alternate executor rejected valid payload: calls=%d err=%v", calls, err) + } + }) + } +} diff --git a/misc/fctl-plugin/plugin.go b/misc/fctl-plugin/plugin.go new file mode 100644 index 0000000000..64c0d6f4f5 --- /dev/null +++ b/misc/fctl-plugin/plugin.go @@ -0,0 +1,211 @@ +package ledger + +import ( + "cmp" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/formancehq/fctl/pkg/pluginsdk" + httpclient "github.com/formancehq/fctl/pkg/pluginsdk/httpclient" +) + +const defaultServiceVersion = "3.0.0" + +type plugin struct { + execute func(context.Context, preparedRequest) (pluginsdk.ExecuteResponse, error) + prepareEndpoint func(string) (*httpclient.Client, error) + serviceVersion string +} + +type preparedRequest struct { + request pluginsdk.ExecuteRequest + op operation + path string + headers http.Header + query url.Values + client *httpclient.Client +} + +// New creates a Ledger plugin with the caller's HTTP transport. Authentication, +// endpoint selection and file/stdin handling belong to the host. +func New(httpClient *http.Client) pluginsdk.Plugin { + return NewVersion(httpClient, defaultServiceVersion) +} + +// NewVersion creates a plugin for the co-released Ledger service version. +// The version belongs to this instance; callers cannot mutate other instances. +func NewVersion(httpClient *http.Client, serviceVersion string) pluginsdk.Plugin { + if httpClient == nil { + httpClient = http.DefaultClient + } + + return &plugin{ + execute: executeHTTP, + prepareEndpoint: func(endpoint string) (*httpclient.Client, error) { + return httpclient.New(endpoint, httpClient) + }, + serviceVersion: cmp.Or(serviceVersion, defaultServiceVersion), + } +} + +// NewWithExecutor reuses the Ledger manifest and command validation with an +// alternate transport. The executor receives normalized flags and an already +// prepared JSON Body; it owns transport-specific endpoint and payload decoding. +// The host retains authentication, file/stdin handling, prompts and rendering. +// The executor must honor ctx and return partial data alongside errors when +// applicable. It is called once per execution, without automatic retries. +// A nil executor permits manifest inspection but cannot execute commands. +func NewWithExecutor(serviceVersion string, executor func(context.Context, pluginsdk.ExecuteRequest) (pluginsdk.ExecuteResponse, error)) pluginsdk.Plugin { + p := &plugin{serviceVersion: cmp.Or(serviceVersion, defaultServiceVersion)} + if executor != nil { + p.execute = func(ctx context.Context, req preparedRequest) (pluginsdk.ExecuteResponse, error) { + return executor(ctx, req.request) + } + } + + return p +} + +func (p *plugin) GetManifest(ctx context.Context) (pluginsdk.Manifest, error) { + if err := ctx.Err(); err != nil { + return pluginsdk.Manifest{}, err + } + manifest, _ := buildLayout() + manifest.Version = p.serviceVersion + + return manifest, nil +} + +func (p *plugin) Execute(ctx context.Context, req pluginsdk.ExecuteRequest) (pluginsdk.ExecuteResponse, error) { + if p.prepareEndpoint == nil { + if err := ctx.Err(); err != nil { + return pluginsdk.ExecuteResponse{}, err + } + } + manifest, operations := buildLayout() + manifest.Version = p.serviceVersion + req, err := pluginsdk.NormalizeRequest(manifest, req) + if err != nil { + return pluginsdk.ExecuteResponse{}, err + } + op := operations[strings.Join(req.CommandPath, "/")] + + prepared, err := p.prepareRequest(op, req) + if err != nil { + return pluginsdk.ExecuteResponse{}, err + } + if p.execute == nil { + return pluginsdk.ExecuteResponse{}, errors.New("ledger command execution requires an executor") + } + + return p.execute(ctx, prepared) +} + +func (p *plugin) prepareRequest(op operation, req pluginsdk.ExecuteRequest) (preparedRequest, error) { + if err := validateArgs(op, req.Args); err != nil { + return preparedRequest{}, err + } + path, err := requestPath(op, req.Args, req.Flags["ledger"]) + if err != nil { + return preparedRequest{}, err + } + headers, err := requestHeaders(op, req.Flags) + if err != nil { + return preparedRequest{}, err + } + req, err = normalizePagination(op, req) + if err != nil { + return preparedRequest{}, err + } + query, err := requestQuery(op, req) + if err != nil { + return preparedRequest{}, err + } + // Preserve the HTTP adapter's endpoint validation order without requiring + // an alternate executor to use an HTTP URL. + var client *httpclient.Client + if p.prepareEndpoint != nil { + client, err = p.prepareEndpoint(req.Endpoint) + if err != nil { + return preparedRequest{}, err + } + } + body, err := requestBody(op, req) + if err != nil { + return preparedRequest{}, err + } + if len(body) > 0 && op.validateBody != nil { + if err := op.validateBody(body, "body"); err != nil { + return preparedRequest{}, fmt.Errorf("invalid request body: %w", err) + } + } + req.Body = body + + return preparedRequest{request: req, op: op, path: path, headers: headers, query: query, client: client}, nil +} + +func executeHTTP(ctx context.Context, req preparedRequest) (pluginsdk.ExecuteResponse, error) { + result, err := req.client.Do(ctx, req.op.method, req.path, req.query, req.request.Body, req.headers) + response := pluginsdk.ExecuteResponse{Data: result} + if err != nil { + if !req.op.bulk { + return pluginsdk.ExecuteResponse{}, err + } + if failure, ok := errors.AsType[*httpclient.Error](err); ok { + response.Data = failure.Body + } + + return response, err + } + if req.op.bulk { + return response, bulkError(result) + } + + return response, nil +} + +func requestBody(op operation, req pluginsdk.ExecuteRequest) (json.RawMessage, error) { + if op.body == bodyNone { + return nil, nil + } + if len(req.Body) > 0 { + return req.Body, nil + } + if op.body == bodyDefault && req.Flags["data"] == "{}" { + return json.RawMessage("{}"), nil + } + if op.body == bodyRequired || req.ChangedFlags["data"] { + return nil, errors.New("host must supply the already-read JSON request body") + } + + return nil, nil +} + +// bulkError inspects only status fields; payload numbers stay raw JSON. +func bulkError(result json.RawMessage) error { + var response struct { + ErrorCode string `json:"errorCode"` + Data []struct { + ErrorCode string `json:"errorCode"` + ResponseType string `json:"responseType"` + } `json:"data"` + } + if err := json.Unmarshal(result, &response); err != nil { + return fmt.Errorf("invalid bulk response: %w", err) + } + if response.ErrorCode != "" { + return fmt.Errorf("bulk failed (%s); inspect the JSON response", response.ErrorCode) + } + for i, item := range response.Data { + if item.ErrorCode != "" || item.ResponseType == "ERROR" { + return fmt.Errorf("bulk element %d failed (%s); inspect the JSON response", i, item.ErrorCode) + } + } + + return nil +} diff --git a/misc/fctl-plugin/plugin_test.go b/misc/fctl-plugin/plugin_test.go new file mode 100644 index 0000000000..71cca54186 --- /dev/null +++ b/misc/fctl-plugin/plugin_test.go @@ -0,0 +1,339 @@ +package ledger + +import ( + "context" + "encoding/base64" + "errors" + "io" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "strings" + "testing" + + "github.com/formancehq/fctl/pkg/pluginsdk" +) + +type request struct { + method, path, body string + query url.Values + header http.Header +} + +const envelope = `{"data":{"amount":1234567890123456789012345678901234567890,"balance":"1234567890123456789012345678901234567890"},"next":"next/+=","previous":"previous","hasMore":true}` + +func execute(t *testing.T, args []string, input, response string, status int) (string, []request, error) { + t.Helper() + requests := make(chan request, 10) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("read body: %v", err) + } + requests <- request{r.Method, r.URL.EscapedPath(), string(body), r.URL.Query(), r.Header.Clone()} + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if _, err := io.WriteString(w, response); err != nil { + t.Errorf("write response: %v", err) + } + })) + t.Cleanup(server.Close) + p := New(server.Client()) + manifest, err := p.GetManifest(t.Context()) + if err != nil { + t.Fatal(err) + } + req := testRequest(manifest, args, input) + req.Endpoint = server.URL + "/gateway/ledger" + result, err := p.Execute(t.Context(), req) + var got []request + for len(requests) > 0 { + got = append(got, <-requests) + } + + return string(result.Data), got, err +} + +type routeCase struct { + name, method, path string + args []string + body string +} + +func TestV3Routes(t *testing.T) { + t.Parallel() + cases := []routeCase{ + {"ledgers", "GET", "/v3/", []string{"list"}, ""}, + {"create", "POST", "/v3/books", []string{"create", "books"}, "{}"}, + {"create selected", "POST", "/v3/books", []string{"--ledger", "books", "create", "--data", `{"metadata":{"owner":"test"}}`}, `{"metadata":{"owner":"test"}}`}, + {"show", "GET", "/v3/books", []string{"show", "books"}, ""}, + {"show selected", "GET", "/v3/books", []string{"--ledger", "books", "show"}, ""}, + {"delete", "DELETE", "/v3/books", []string{"delete", "books", "--confirm"}, ""}, + {"stats", "GET", "/v3/books/stats", []string{"stats", "--ledger", "books"}, ""}, + {"info", "GET", "/_info", []string{"info"}, ""}, + {"accounts", "GET", "/v3/books/accounts", []string{"accounts", "list"}, ""}, + {"account", "GET", "/v3/books/accounts/users:42", []string{"accounts", "show", "users:42"}, ""}, + {"account balances", "GET", "/v3/books/accounts/users:42", []string{"accounts", "balances", "users:42"}, ""}, + {"balances", "GET", "/v3/books/volumes", []string{"balances"}, ""}, + {"transactions", "GET", "/v3/books/transactions", []string{"transactions", "list"}, ""}, + {"transaction", "GET", "/v3/books/transactions/18446744073709551615", []string{"transactions", "show", "18446744073709551615"}, ""}, + {"transaction create", "POST", "/v3/books/transactions", []string{"transactions", "create", "--data", `{"postings":[{"source":"world","destination":"users:42","asset":"USD/2","amount":123456789012345678901234567890}]}`}, `{"postings":[{"source":"world","destination":"users:42","asset":"USD/2","amount":123456789012345678901234567890}]}`}, + {"revert", "POST", "/v3/books/transactions/42/revert", []string{"transactions", "revert", "42"}, ""}, + {"revert body", "POST", "/v3/books/transactions/42/revert", []string{"transactions", "revert", "42", "--data", `{"metadata":{"reason":"test"}}`}, `{"metadata":{"reason":"test"}}`}, + {"ledger metadata read", "GET", "/v3/books", []string{"metadata", "show"}, ""}, + {"ledger metadata set", "POST", "/v3/books/metadata", []string{"metadata", "set", "--data", `{"score":9007199254740993}`}, `{"score":9007199254740993}`}, + {"ledger metadata delete", "DELETE", "/v3/books/metadata/formance.com%2Freviewed", []string{"metadata", "delete", "formance.com/reviewed", "--confirm"}, ""}, + {"account metadata read", "GET", "/v3/books/accounts/users:42", []string{"accounts", "metadata", "show", "users:42"}, ""}, + {"account metadata set", "POST", "/v3/books/accounts/users:42/metadata", []string{"accounts", "metadata", "set", "users:42", "--data", `{"score":9007199254740993}`}, `{"score":9007199254740993}`}, + {"account metadata delete", "DELETE", "/v3/books/accounts/users:42/metadata/formance.com%2Freviewed", []string{"accounts", "metadata", "delete", "users:42", "formance.com/reviewed", "--confirm"}, ""}, + {"transaction metadata read", "GET", "/v3/books/transactions/42", []string{"transactions", "metadata", "show", "42"}, ""}, + {"transaction metadata set", "POST", "/v3/books/transactions/42/metadata", []string{"transactions", "metadata", "set", "42", "--data", `{"score":9007199254740993}`}, `{"score":9007199254740993}`}, + {"transaction metadata delete", "DELETE", "/v3/books/transactions/42/metadata/a%2Fb", []string{"transactions", "metadata", "delete", "42", "a/b", "--confirm"}, ""}, + {"logs", "GET", "/v3/books/logs", []string{"logs", "list"}, ""}, + {"bulk", "POST", "/v3/books/bulk", []string{"bulk", "--data", `[{"action":"CREATE_TRANSACTION","ik":"payment-42","data":{"postings":[]}}]`}, `[{"action":"CREATE_TRANSACTION","ik":"payment-42","data":{"postings":[]}}]`}, + {"indexes", "GET", "/v3/books/indexes", []string{"indexes", "list"}, ""}, + {"index", "GET", "/v3/books/indexes/metadata:TARGET_TYPE_ACCOUNT:a%2Fb", []string{"indexes", "show", "metadata:TARGET_TYPE_ACCOUNT:a/b"}, ""}, + {"index status", "GET", "/v3/books/indexes/metadata:TARGET_TYPE_ACCOUNT:a%2Fb/status", []string{"indexes", "status", "metadata:TARGET_TYPE_ACCOUNT:a/b"}, ""}, + {"index create", "POST", "/v3/books/indexes", []string{"indexes", "create", "--data", `{"id":"metadata:TARGET_TYPE_ACCOUNT:a/b"}`}, `{"id":"metadata:TARGET_TYPE_ACCOUNT:a/b"}`}, + {"index delete", "DELETE", "/v3/books/indexes/metadata:TARGET_TYPE_ACCOUNT:a%2Fb", []string{"indexes", "delete", "metadata:TARGET_TYPE_ACCOUNT:a/b", "--confirm"}, ""}, + {"index inspect", "GET", "/v3/books/indexes/metadata:TARGET_TYPE_ACCOUNT:a%2Fb/inspect", []string{"indexes", "inspect", "metadata:TARGET_TYPE_ACCOUNT:a/b"}, ""}, + {"raw ledger", "GET", "/v3/a%2Fb%25%20c", []string{"show", "a/b% c"}, ""}, + {"raw address", "GET", "/v3/books/accounts/users:a%2Fb%25%20c", []string{"accounts", "show", "users:a/b% c"}, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + checkRoute(t, tc) + }) + } +} + +func routeArguments(tc routeCase) []string { + args := slices.Clone(tc.args) + unscoped := []string{"ledgers", "info", "raw ledger", "create selected", "show selected", "create", "show", "delete"} + if !slices.Contains(unscoped, tc.name) { + args = append([]string{"--ledger", "books"}, args...) + } + if tc.method != http.MethodGet { + args = append(args, "--idempotency-key", "operation-42") + } + + return args +} + +func checkRoute(t *testing.T, tc routeCase) { + t.Helper() + response := envelope + if tc.name == "bulk" { + response = strings.Replace(envelope, `"data":{`, `"data":[{`, 1) + response = strings.Replace(response, `},"next"`, `}],"next"`, 1) + } + out, requests, err := execute(t, routeArguments(tc), "", response, 200) + if err != nil { + t.Fatal(err) + } + if len(requests) != 1 { + t.Fatalf("requests = %d, want exactly one", len(requests)) + } + checkRequest(t, requests[0], tc) + for _, value := range []string{"1234567890123456789012345678901234567890", `"next"`, `"previous"`, `"hasMore"`} { + if !strings.Contains(out, value) { + t.Errorf("output lost %s: %s", value, out) + } + } +} + +func checkRequest(t *testing.T, got request, tc routeCase) { + t.Helper() + if got.method != tc.method || got.path != "/gateway/ledger"+tc.path || got.body != tc.body { + t.Fatalf("request = %#v; want %s %s body %s", got, tc.method, tc.path, tc.body) + } + if tc.body != "" && got.header.Get("Content-Type") != "application/json" { + t.Error("missing JSON content type") + } + if tc.method != http.MethodGet && got.header.Get("Idempotency-Key") != "operation-42" { + t.Error("write lost its idempotency identity") + } +} + +func TestQueryAndHeaders(t *testing.T) { + t.Parallel() + for _, collection := range []string{"transactions", "logs", "accounts"} { + t.Run(collection, func(t *testing.T) { checkListQuery(t, collection) }) + } +} + +func checkListQuery(t *testing.T, collection string) { + t.Helper() + after := "18446744073709551615" + if collection == "accounts" { + after = "users:next/+=" + } + args := []string{"--ledger", "books", "--consistency", " STALE ", collection, "list", "--page-size", "0", "--after", after, "--filter", `metadata[score] >= 9007199254740993`} + cursor := base64.RawURLEncoding.EncodeToString([]byte(`{"key":"` + after + `"}`)) + want := map[string]string{"pageSize": "0", "cursor": cursor, "filter": `metadata[score] >= 9007199254740993`} + if collection != "logs" { + args = append(args, "--reverse") + want["reverse"] = "true" + } + if collection != "accounts" { + args = append(args, "--start-date", "2026-10-01T00:00:00Z", "--end-date", "2026-10-08T12:00:00+02:00") + want["startDate"] = "2026-10-01T00:00:00Z" + want["endDate"] = "2026-10-08T12:00:00+02:00" + } + _, reqs, err := execute(t, args, "", envelope, 200) + if err != nil { + t.Fatal(err) + } + checkQuery(t, reqs[0].query, want) + if len(reqs[0].query) != len(want) || reqs[0].query.Has("after") || (collection == "logs" && reqs[0].query.Has("reverse")) { + t.Fatalf("unsupported query keys: %v", reqs[0].query) + } + if reqs[0].header.Get("X-Consistency") != "stale" { + t.Error("missing read consistency") + } +} + +func checkQuery(t *testing.T, got url.Values, want map[string]string) { + t.Helper() + for key, value := range want { + if got.Get(key) != value { + t.Errorf("query %s = %q, want %q", key, got.Get(key), value) + } + } +} + +func TestAggregateQuery(t *testing.T) { + t.Parallel() + _, reqs, err := execute(t, []string{"--ledger", "books", "balances", "--collapse-colors", "--use-max-precision", "--group-by-prefixes", "users:,merchants:", "--filter", `{"$match":{"address":"users:42"}}`}, "", envelope, 200) + if err != nil { + t.Fatal(err) + } + checkQuery(t, reqs[0].query, map[string]string{"collapseColors": "true", "useMaxPrecision": "true", "groupByPrefixes": "users:,merchants:", "filter": `{"$match":{"address":"users:42"}}`}) +} + +func TestIndexInspectionQuery(t *testing.T) { + t.Parallel() + _, reqs, err := execute(t, []string{"--ledger", "books", "indexes", "inspect", "metadata:TARGET_TYPE_ACCOUNT:key", "--mode", "facets", "--page-size", "10000", "--cursor", "previous/+="}, "", envelope, 200) + if err != nil { + t.Fatal(err) + } + checkQuery(t, reqs[0].query, map[string]string{"mode": "facets", "pageSize": "10000", "cursor": "previous/+="}) + if reqs[0].query.Has("after") { + t.Fatalf("index inspection sent entity pagination: %v", reqs[0].query) + } +} + +func TestBodiesAndIdempotency(t *testing.T) { + t.Parallel() + body := `{"postings":[],"metadata":{"number":9007199254740993}}` + // The host already read all three inputs. The plugin only receives JSON. + for _, data := range []string{body, "@already-read.json", "-"} { + _, reqs, err := execute(t, []string{"--ledger", "books", "transactions", "create", "--data", data, "--idempotency-key", "payment-42"}, body, envelope, 201) + if err != nil { + t.Fatal(err) + } + if len(reqs) != 1 || reqs[0].body != body || reqs[0].header.Get("Idempotency-Key") != "payment-42" { + t.Fatalf("body or identity changed: %#v", reqs) + } + } + _, reqs, err := execute(t, []string{"--ledger", "books", "bulk", "--data", "[]", "--atomic", "--continue-on-failure=false", "--idempotency-key", "batch-42"}, "", `{"data":[]}`, 200) + if err != nil || reqs[0].query.Get("atomic") != "true" || reqs[0].query.Get("continueOnFailure") != "false" || reqs[0].header.Get("Idempotency-Key") != "batch-42" { + t.Fatalf("bulk: %v, %v", reqs, err) + } +} + +func TestValidationBeforeHTTP(t *testing.T) { + t.Parallel() + cases := [][]string{ + {"accounts", "list"}, {"create"}, {"--ledger", "books", "show", "other"}, + {"--ledger", "books", "accounts", "show"}, {"--ledger", "books", "accounts", "show", ""}, + {"--ledger", "books", "transactions", "show", "-1"}, {"--ledger", "books", "transactions", "show", "18446744073709551616"}, + {"--ledger", "books", "transactions", "metadata", "set", "abc", "--data", "{}"}, + {"--ledger", "books", "transactions", "create"}, {"--ledger", "books", "transactions", "create", "--data", "{} {}"}, + {"--ledger", "books", "transactions", "revert", "42", "--data", ""}, + {"--ledger", "books", "transactions", "create", "--data", "@/nonexistent/fctl/request.json"}, + {"delete", "books"}, {"delete", "books", "--confirm=false"}, + {"--ledger", "books", "metadata", "delete", "key"}, {"--ledger", "books", "indexes", "delete", "index"}, + {"--ledger", "books", "accounts", "list", "--page-size", "-1"}, + {"--ledger", "books", "transactions", "list", "--start-date", "yesterday"}, + {"--ledger", "books", "logs", "list", "--end-date", "1969-12-31T00:00:00Z"}, + {"--ledger", "books", "indexes", "inspect", "index", "--mode", "bad"}, + {"--ledger", "books", "indexes", "inspect", "index", "--page-size", "0"}, + {"--ledger", "books", "indexes", "inspect", "index", "--page-size", "10001"}, + {"--ledger", "books", "--consistency", "eventual", "stats"}, + {"create", "books", "--idempotency-key", strings.Repeat("a", 257)}, + {"create", "books", "--idempotency-key", "a\nb"}, + } + for _, args := range cases { + t.Run(strings.Join(args, " "), func(t *testing.T) { + _, reqs, err := execute(t, args, "", envelope, 200) + if err == nil || len(reqs) != 0 { + t.Fatalf("want validation error before HTTP, got requests=%v err=%v", reqs, err) + } + }) + } +} + +func TestHTTPFailuresNeverRetryMutations(t *testing.T) { + t.Parallel() + for _, status := range []int{400, 401, 403, 404, 409, 429, 500, 503} { + out, reqs, err := execute(t, []string{"--ledger", "books", "transactions", "create", "--data", "{}", "--idempotency-key", "payment-42"}, "", `{"errorCode":"TEST_ERROR","errorMessage":"write refused"}`, status) + if err == nil || !strings.Contains(err.Error(), "TEST_ERROR") || !strings.Contains(err.Error(), "write refused") || len(reqs) != 1 || out != "" { + t.Fatalf("status %d: output=%s requests=%d err=%v", status, out, len(reqs), err) + } + } + _, reqs, err := execute(t, []string{"show", "books"}, "", "not json", 200) + if err == nil || len(reqs) != 1 { + t.Fatalf("invalid response: requests=%d err=%v", len(reqs), err) + } + out, reqs, err := execute(t, []string{"delete", "books", "--confirm", "--idempotency-key", "delete-books"}, "", "", 204) + if err != nil || out != "null" || len(reqs) != 1 || reqs[0].header.Get("Idempotency-Key") != "delete-books" { + t.Fatalf("204 response: output=%s requests=%v err=%v", out, reqs, err) + } +} + +func TestEndpointAndContextErrors(t *testing.T) { + t.Parallel() + p := New(nil) + _, err := p.Execute(t.Context(), pluginsdk.ExecuteRequest{CommandPath: []string{"ledger", "list"}, Endpoint: "invalid"}) + if err == nil { + t.Fatal("invalid endpoint accepted") + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := p.GetManifest(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("manifest context error = %v", err) + } +} + +func TestBulkBusinessFailuresAreNonzero(t *testing.T) { + t.Parallel() + for _, response := range []string{ + `{"data":[{"responseType":"CREATE_TRANSACTION","data":{"id":42}},{"responseType":"ERROR","errorCode":"INSUFFICIENT_FUNDS","errorDescription":"not enough funds"}]}`, + `{"data":[],"errorCode":"BATCH_FAILED","errorMessage":"batch refused"}`, + `{"data":{}}`, + } { + out, reqs, err := execute(t, []string{"--ledger", "books", "bulk", "--data", "[]", "--continue-on-failure"}, "", response, 200) + if err == nil || len(reqs) != 1 || out == "" { + t.Fatalf("bulk failures must keep JSON and fail once: out=%s requests=%d err=%v", out, len(reqs), err) + } + } + _, reqs, err := execute(t, []string{"--ledger", "books", "bulk", "--data", "[]"}, "", `{"errorCode":"UNAVAILABLE","errorMessage":"leader lost"}`, 503) + if err == nil || len(reqs) != 1 { + t.Fatalf("bulk HTTP failure retried or ignored: requests=%d err=%v", len(reqs), err) + } +} + +func TestPageSizeUsesV3ServerSemantics(t *testing.T) { + t.Parallel() + _, reqs, err := execute(t, []string{"--ledger", "books", "accounts", "list", "--page-size", "1001", "--reverse=false", "--consistency", "linearizable"}, "", `{"data":[],"hasMore":false}`, 200) + if err != nil || reqs[0].query.Get("pageSize") != "1001" || reqs[0].query.Get("reverse") != "false" || reqs[0].header.Get("X-Consistency") != "linearizable" { + t.Fatalf("page options = %v, err=%v", reqs, err) + } + _, reqs, err = execute(t, []string{"accounts", "balances", "users:42", "--ledger", "books", "--collapse-colors=false"}, "", envelope, 200) + if err != nil || reqs[0].query.Get("collapseColors") != "false" { + t.Fatalf("account options = %v, err=%v", reqs, err) + } +} diff --git a/misc/fctl-plugin/request.go b/misc/fctl-plugin/request.go new file mode 100644 index 0000000000..03ee733f47 --- /dev/null +++ b/misc/fctl-plugin/request.go @@ -0,0 +1,305 @@ +package ledger + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "maps" + "net/http" + "net/url" + "slices" + "strconv" + "strings" + "time" + + "github.com/formancehq/fctl/pkg/pluginsdk" +) + +type bodyMode uint8 + +const ( + bodyNone bodyMode = iota + bodyRequired + bodyOptional + bodyDefault +) + +type operation struct { + use, short, method string + args int + global, ledgerArg bool + transactionID bool + path func([]string) []string + body bodyMode + validateBody payloadValidator + idempotency bool + page, reverse bool + afterID bool + filter, dates bool + inspect, bulk bool + boolQuery map[string]string + stringQuery map[string]string +} + +func (layout) endpoint(op operation) *node { + if op.method == "" { + op.method = http.MethodGet + } + spec := pluginsdk.CommandSpec{Use: op.use, Short: op.short, Args: pluginsdk.ArgsSpec{Min: op.args, Max: op.args}, Runnable: true, Confirm: op.method == http.MethodDelete} + if op.ledgerArg { + spec.Args = pluginsdk.ArgsSpec{Min: 0, Max: 1} + } + spec.Flags = append(bodyFlags(op), queryFlags(op)...) + if !op.global { + spec.Inputs = []pluginsdk.InputSpec{ledgerInput(op)} + } + + return &node{spec: spec, op: &op} +} + +func bodyFlags(op operation) []pluginsdk.FlagSpec { + var flags []pluginsdk.FlagSpec + if op.body != bodyNone { + value := "" + if op.body == bodyDefault { + value = "{}" + } + flags = append(flags, pluginsdk.FlagSpec{Name: "data", Type: "string", Default: value, Body: true, Required: op.body == bodyRequired, Usage: "JSON request body: inline JSON, @file or - for stdin"}) + } + if op.idempotency { + flags = append(flags, pluginsdk.FlagSpec{Name: "idempotency-key", Type: "string", Usage: "Stable request identity (max 256 bytes); never generated automatically"}) + } + if op.method == http.MethodDelete { + flags = append(flags, pluginsdk.FlagSpec{Name: "confirm", Type: "bool", Default: "false", Usage: "Confirm this destructive deletion"}) + } + + return flags +} + +func queryFlags(op operation) []pluginsdk.FlagSpec { + var flags []pluginsdk.FlagSpec + if op.page || op.inspect { + usage := "Results per page (0 means 100; server caps at 1000)" + if op.inspect { + usage = "Results per index page (1 through 10000)" + } + flags = append(flags, pluginsdk.FlagSpec{Name: "page-size", Type: "uint32", Default: "100", Usage: usage}) + flags = append(flags, pluginsdk.FlagSpec{Name: "cursor", Type: "string", Usage: "Opaque next or previous page token returned by the server"}) + if !op.inspect && !op.global { + afterUsage := "Compatibility alias for --cursor: continue after this account address (exclusive)" + if op.afterID { + afterUsage = "Compatibility alias for --cursor: continue after this unsigned transaction or ledger-local log ID (exclusive)" + } + flags = append(flags, pluginsdk.FlagSpec{Name: "after", Type: "string", Usage: afterUsage}) + } + } + if op.reverse { + flags = append(flags, pluginsdk.FlagSpec{Name: "reverse", Type: "bool", Default: "false", Usage: "Reverse the endpoint's default ordering"}) + } + if op.filter { + flags = append(flags, pluginsdk.FlagSpec{Name: "filter", Type: "string", Usage: "Exact v3 filter expression or structured JSON filter"}) + } + if op.dates { + flags = append(flags, pluginsdk.FlagSpec{Name: "start-date", Type: "string", Usage: "Inclusive start date (RFC3339)"}, pluginsdk.FlagSpec{Name: "end-date", Type: "string", Usage: "Exclusive end date (RFC3339)"}) + } + if op.inspect { + flags = append(flags, pluginsdk.FlagSpec{Name: "mode", Type: "string", Default: "summary", Usage: "Index inspection: summary, distinctValues or facets"}) + } + for _, name := range slices.Sorted(maps.Keys(op.boolQuery)) { + flags = append(flags, pluginsdk.FlagSpec{Name: name, Type: "bool", Default: "false", Usage: "Set the v3 " + op.boolQuery[name] + " query option"}) + } + for _, name := range slices.Sorted(maps.Keys(op.stringQuery)) { + flags = append(flags, pluginsdk.FlagSpec{Name: name, Type: "string", Usage: "Set the v3 " + op.stringQuery[name] + " query option"}) + } + + return flags +} + +func validateArgs(op operation, args []string) error { + if slices.Contains(args, "") { + return errors.New("resource identifiers must not be empty") + } + if op.transactionID { + if _, err := strconv.ParseUint(args[0], 10, 64); err != nil { + return fmt.Errorf("transaction id must be an unsigned 64-bit integer: %w", err) + } + } + + return nil +} + +func requestHeaders(op operation, flags map[string]string) (http.Header, error) { + if op.method == http.MethodDelete && flags["confirm"] != "true" { + return nil, errors.New("deletion requires --confirm") + } + if len(flags["idempotency-key"]) > 256 || strings.ContainsAny(flags["idempotency-key"], "\r\n") { + return nil, errors.New("idempotency-key must be at most 256 bytes without newlines") + } + headers := make(http.Header) + if flags["idempotency-key"] != "" { + headers.Set("Idempotency-Key", flags["idempotency-key"]) + } + if flags["consistency"] != "" { + consistency := strings.ToLower(strings.TrimSpace(flags["consistency"])) + if consistency != "linearizable" && consistency != "stale" { + return nil, errors.New("consistency must be linearizable or stale") + } + if !op.global || op.page { + headers.Set("X-Consistency", consistency) + } + } + + return headers, nil +} + +func requestQuery(op operation, req pluginsdk.ExecuteRequest) (url.Values, error) { + query := make(url.Values) + if err := paginationQuery(query, op, req.Flags); err != nil { + return nil, err + } + if op.reverse && req.ChangedFlags["reverse"] { + query.Set("reverse", req.Flags["reverse"]) + } + if req.Flags["filter"] != "" { + query.Set("filter", req.Flags["filter"]) + } + if err := dateQuery(query, req.Flags); err != nil { + return nil, err + } + if op.inspect { + if err := inspectQuery(query, req.Flags); err != nil { + return nil, err + } + } + copyOptions(req, query, op.boolQuery) + copyOptions(req, query, op.stringQuery) + + return query, nil +} + +func paginationQuery(query url.Values, op operation, flags map[string]string) error { + if op.page || op.inspect { + query.Set("pageSize", flags["page-size"]) + if flags["cursor"] != "" { + query.Set("cursor", flags["cursor"]) + } + } + + return nil +} + +// normalizePagination prepares the same forward token for HTTP and alternate +// executors. --after is consumed before dispatch; cursor is the canonical input. +// Index inspection's value tokens remain opaque here. +func normalizePagination(op operation, req pluginsdk.ExecuteRequest) (pluginsdk.ExecuteRequest, error) { + if !op.page { + return req, nil + } + if after := req.Flags["after"]; after != "" { + if req.Flags["cursor"] != "" { + return req, errors.New("--after and --cursor cannot be used together") + } + if op.afterID { + id, err := strconv.ParseUint(after, 10, 64) + if err != nil { + return req, fmt.Errorf("after must be an unsigned 64-bit transaction or log ID: %w", err) + } + after = strconv.FormatUint(id, 10) + } + token, err := json.Marshal(struct { + Key string `json:"key"` + }{Key: after}) + if err != nil { + return req, fmt.Errorf("encode after cursor: %w", err) + } + req.Flags["cursor"] = base64.RawURLEncoding.EncodeToString(token) + req.ChangedFlags = maps.Clone(req.ChangedFlags) + if req.ChangedFlags == nil { + req.ChangedFlags = make(map[string]bool) + } + req.ChangedFlags["cursor"] = true + delete(req.Flags, "after") + delete(req.ChangedFlags, "after") + } + if token := req.Flags["cursor"]; token != "" { + if err := validatePageCursor(token, op.afterID); err != nil { + return req, fmt.Errorf("invalid --cursor: %w", err) + } + } + + return req, nil +} + +func validatePageCursor(token string, numeric bool) error { + raw, err := base64.RawURLEncoding.DecodeString(token) + if err != nil { + return fmt.Errorf("expected a base64url page token: %w", err) + } + fields, err := payloadObject(raw, "cursor") + if err != nil { + return err + } + var key string + for name, value := range fields { + switch name { + case "key": + if bytes.Equal(bytes.TrimSpace(value), []byte("null")) { + return errors.New("cursor.key must be a string") + } + if err := json.Unmarshal(value, &key); err != nil { + return fmt.Errorf("cursor.key must be a string: %w", err) + } + case "back": + if err := validateJSONBool(value, "cursor.back"); err != nil { + return err + } + default: + return fmt.Errorf("unknown cursor field %q", name) + } + } + if numeric && key != "" { + if _, err := strconv.ParseUint(key, 10, 64); err != nil { + return fmt.Errorf("cursor.key must be an unsigned 64-bit transaction or log ID: %w", err) + } + } + + return nil +} + +func dateQuery(query url.Values, flags map[string]string) error { + for name, value := range map[string]string{"startDate": flags["start-date"], "endDate": flags["end-date"]} { + if value == "" { + continue + } + date, err := time.Parse(time.RFC3339Nano, value) + if err != nil || date.Before(time.Unix(0, 0)) { + return fmt.Errorf("%s must be an RFC3339 date on or after the Unix epoch", name) + } + query.Set(name, value) + } + + return nil +} + +func inspectQuery(query url.Values, flags map[string]string) error { + if flags["mode"] != "summary" && flags["mode"] != "distinctValues" && flags["mode"] != "facets" { + return errors.New("mode must be summary, distinctValues or facets") + } + size, err := strconv.ParseUint(flags["page-size"], 10, 32) + if err != nil || size < 1 || size > 10000 { + return errors.New("index page-size must be between 1 and 10000") + } + query.Set("mode", flags["mode"]) + + return nil +} + +func copyOptions(req pluginsdk.ExecuteRequest, query url.Values, options map[string]string) { + for flag, param := range options { + if req.ChangedFlags[flag] { + query.Set(param, req.Flags[flag]) + } + } +} diff --git a/misc/fctl-plugin/test_request_test.go b/misc/fctl-plugin/test_request_test.go new file mode 100644 index 0000000000..aee8ef663b --- /dev/null +++ b/misc/fctl-plugin/test_request_test.go @@ -0,0 +1,79 @@ +package ledger + +import ( + "encoding/json" + "strings" + + "github.com/formancehq/fctl/pkg/pluginsdk" +) + +// testRequest expresses the previous command fixtures as SDK requests. It only +// separates their tokens; validation and defaults belong to NormalizeRequest. +// File and stdin inputs are supplied as already-read JSON by the calling test. +func testRequest(manifest pluginsdk.Manifest, tokens []string, body string) pluginsdk.ExecuteRequest { + kinds := make(map[string]string) + collectFlags(manifest.Root, kinds) + req := pluginsdk.ExecuteRequest{CommandPath: []string{"ledger"}, Flags: make(map[string]string), ChangedFlags: make(map[string]bool)} + current := manifest.Root + for i := 0; i < len(tokens); i++ { + token := tokens[i] + if name, ok := strings.CutPrefix(token, "--"); ok { + flag, value, consumed := testFlag(name, tokens[i+1:], kinds) + i += consumed + req.Flags[flag], req.ChangedFlags[flag] = value, true + + continue + } + if child, found := testChild(current, token); found && len(req.Args) == 0 { + current = child + req.CommandPath = append(req.CommandPath, token) + } else { + req.Args = append(req.Args, token) + } + } + req.Body = testBody(req.Flags, body) + + return req +} + +func collectFlags(spec pluginsdk.CommandSpec, kinds map[string]string) { + for _, flag := range spec.Flags { + kinds[flag.Name] = flag.Type + } + for _, child := range spec.Subcommands { + collectFlags(child, kinds) + } +} + +func testChild(spec pluginsdk.CommandSpec, name string) (pluginsdk.CommandSpec, bool) { + for _, child := range spec.Subcommands { + if strings.Fields(child.Use)[0] == name { + return child, true + } + } + + return pluginsdk.CommandSpec{}, false +} + +func testFlag(name string, remaining []string, kinds map[string]string) (string, string, int) { + flag, value, inline := strings.Cut(name, "=") + if inline { + return flag, value, 0 + } + if kinds[flag] != "bool" && len(remaining) > 0 { + return flag, remaining[0], 1 + } + + return flag, "true", 0 +} + +func testBody(flags map[string]string, body string) json.RawMessage { + if body != "" { + return json.RawMessage(body) + } + if data, changed := flags["data"]; changed && !strings.HasPrefix(data, "@") && data != "-" { + return json.RawMessage(data) + } + + return nil +} diff --git a/misc/fctl-plugin/version_test.go b/misc/fctl-plugin/version_test.go new file mode 100644 index 0000000000..d5d71920f7 --- /dev/null +++ b/misc/fctl-plugin/version_test.go @@ -0,0 +1,26 @@ +package ledger + +import "testing" + +func TestServiceVersionBelongsToPluginInstance(t *testing.T) { + t.Parallel() + for _, version := range []string{"3.0.0", "3.0.1", "3.0.0-beta.5"} { + t.Run(version, func(t *testing.T) { + t.Parallel() + plugin := NewVersion(nil, version) + manifest, err := plugin.GetManifest(t.Context()) + if err != nil { + t.Fatal(err) + } + if manifest.Version != version { + t.Fatalf("manifest version: %q, want %q", manifest.Version, version) + } + manifest.Version = "corrupted" + manifest.Root.Flags[0].Name = "corrupted" + fresh, err := plugin.GetManifest(t.Context()) + if err != nil || fresh.Version != version || fresh.Root.Flags[0].Name != "ledger" { + t.Fatalf("manifest mutation escaped caller: %#v, %v", fresh, err) + } + }) + } +} diff --git a/scripts/agent-check b/scripts/agent-check index ef7a2f4489..77e169d72e 100644 --- a/scripts/agent-check +++ b/scripts/agent-check @@ -23,6 +23,9 @@ bash "$SCRIPT_DIR/check-repo-invariants" echo "==> agent-check: compile all root-module packages" go build ./... +echo "==> agent-check: compile the product-owned fctl plugin" +(cd misc/fctl-plugin && go build ./...) + echo "==> agent-check: verify tracked/staged diff whitespace" git diff --check HEAD -- diff --git a/scripts/agent-check-pr b/scripts/agent-check-pr index fda4864a18..9bac895b12 100755 --- a/scripts/agent-check-pr +++ b/scripts/agent-check-pr @@ -77,6 +77,7 @@ needs_e2e=false needs_scenarios=false needs_schemathesis=false needs_operator=false +needs_fctl_plugin=false has_change=false changed_packages=() @@ -143,6 +144,11 @@ classify_path() { needs_operator=true return ;; + misc/fctl-plugin/*) + needs_precommit=true + needs_fctl_plugin=true + return + ;; tests/antithesis/workload/*) needs_precommit=true return @@ -216,6 +222,9 @@ fi if [[ "$needs_operator" == true ]]; then gates+=(test-operator) fi +if [[ "$needs_fctl_plugin" == true ]]; then + gates+=(test-fctl-plugin) +fi if [[ "$mode" == list ]]; then printf '%s\n' "${gates[@]}" @@ -306,6 +315,9 @@ fi if [[ "$needs_operator" == true ]]; then go -C misc/operator test ./... -timeout 20m fi +if [[ "$needs_fctl_plugin" == true ]]; then + bash "$SCRIPT_DIR/agent-just" test-fctl-plugin +fi if [[ "$has_change" == false ]]; then echo "==> agent-check-pr: no candidate changes" diff --git a/scripts/agentcheckpr/selector_test.go b/scripts/agentcheckpr/selector_test.go index 7fbca72f6c..bba969cbe5 100644 --- a/scripts/agentcheckpr/selector_test.go +++ b/scripts/agentcheckpr/selector_test.go @@ -92,6 +92,11 @@ func TestSelectsLocalValidationGatesFromCompleteDiff(t *testing.T) { paths: []string{"misc/operator/internal/example.go"}, expected: []string{"pre-commit", "agent-check", "test-operator"}, }, + { + name: "independent fctl plugin module", + paths: []string{"misc/fctl-plugin/go.mod", "misc/fctl-plugin/plugin.go", "misc/fctl-plugin/cmd/fctl-plugin-ledger/main.go"}, + expected: []string{"pre-commit", "agent-check", "test-fctl-plugin"}, + }, { // Top-level Antithesis files are harness configuration, not a Go // package. Selecting their directory for go test fails with "no Go