From 037244f950624a47341d22e8b142ee82e77e1d43 Mon Sep 17 00:00:00 2001 From: Shelley Shen Date: Mon, 5 Oct 2026 13:47:41 -0700 Subject: [PATCH 01/11] test(bdd): add HTTP request steps and event-ledger feature Add in-process HTTP steps to the BDD harness (headers, send, poll, status, body and JSON field assertions, export to env) so features can exercise service APIs directly without routing secrets through the command runner. Add an event-ledger feature that checks the ledger is deployed, routed through the shared gateway, and fed by SIS on a running ncp-local stack. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Shelley Shen --- tests/bdd/PLAN.md | 19 ++ tests/bdd/dsl/clistate.go | 52 +++++ tests/bdd/dsl/http_test.go | 107 +++++++++ tests/bdd/dsl/jsonfield.go | 100 ++++++++ tests/bdd/features/event-ledger.feature | 298 ++++++++++++++++++++++++ tests/bdd/godog_test.go | 158 +++++++++++++ tests/bdd/harness/http.go | 82 +++++++ tests/bdd/harness/http_test.go | 75 ++++++ tests/bdd/harness/suite.go | 1 + tests/bdd/steps/context.go | 5 + tests/bdd/steps/http_steps.go | 264 +++++++++++++++++++++ tests/bdd/steps/http_steps_test.go | 224 ++++++++++++++++++ 12 files changed, 1385 insertions(+) create mode 100644 tests/bdd/dsl/clistate.go create mode 100644 tests/bdd/dsl/http_test.go create mode 100644 tests/bdd/dsl/jsonfield.go create mode 100644 tests/bdd/features/event-ledger.feature create mode 100644 tests/bdd/harness/http.go create mode 100644 tests/bdd/harness/http_test.go create mode 100644 tests/bdd/steps/http_steps.go create mode 100644 tests/bdd/steps/http_steps_test.go diff --git a/tests/bdd/PLAN.md b/tests/bdd/PLAN.md index 2b5caed9aa..61f5870790 100644 --- a/tests/bdd/PLAN.md +++ b/tests/bdd/PLAN.md @@ -176,6 +176,25 @@ original order. Repeated options and empty values are preserved. | `When I successfully invoke the function selected by NVCF CLI through Vanity Gateway host {string} path {string} with timeout {string} seconds:` (JSON docstring) | Sends an exact-host HTTP request through the local Envoy listener with the saved function API key passed over sensitive stdin, never argv or command logs. | | `When I successfully undeploy the function selected by NVCF CLI` | Runs `function delete --deployment-only`. Function selection remains owned by CLI state. | +### HTTP requests (Given / When / Then) + +These steps use the in-process HTTP client (`harness.HTTPClient`), not the command runner, so headers never reach `out//logs`. Redirects are not followed. Pending headers apply to the next send or poll step only. + +| Step | Notes | +|------|-------| +| `Given the HTTP request header {string} is {string}` | Queues one request header for the next request. `${VAR}` expansion applies to the value. | +| `Given the HTTP request header {string} is the NVCF CLI admin token with prefix {string}` | Queues a header whose value is the prefix plus the admin token stored by `nvcf-cli init` in the state file of the selected NVCF CLI config. The token never enters argv, env, or logs. | +| `When I send a {string} request to {string}` | Sends the request with the queued headers and records status and body. `${VAR}` expansion applies to the URL. A non-2xx status is not a step failure. | +| `When I send a {string} request to {string} with body:` (docstring) | As above with the interpolated docstring as the request body. | +| `When I poll {string} {string} for up to {int} seconds until the HTTP response body contains {string}` | Repeats the request every 5 seconds until the body contains the text or the deadline passes. Used for eventually consistent reads. | +| `Then the HTTP status should be {int}` | Last response status. The failure message includes a truncated body. | +| `Then the HTTP status should be {int} or {int}` | Last response status equals either value. | +| `Then the HTTP response body should contain {string}` | Substring match. The interpolated value must be non-empty. | +| `Then the HTTP response body should not contain {string}` | Negative substring match. The interpolated value must be non-empty. | +| `Then the HTTP response JSON field {string} should equal {string}` | Dotted path (object keys, array indexes) compared as text. | +| `Then the HTTP response JSON field {string} should be at least {int}` | Dotted path parsed as a number and compared to a minimum. | +| `When I export the HTTP response JSON field {string} to environment variable {string}` | Exports the field under the named env var. Fails when the field is missing or empty. Snapshotted by the env Ledger; restored at suite teardown. The value is never echoed. | + ### Assertions (Then / And) | Step | Notes | diff --git a/tests/bdd/dsl/clistate.go b/tests/bdd/dsl/clistate.go new file mode 100644 index 0000000000..92d689a857 --- /dev/null +++ b/tests/bdd/dsl/clistate.go @@ -0,0 +1,52 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dsl + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +// CLIStatePath returns the nvcf-cli state file for a --config path, mirroring +// the CLI: ~/.nvcf-cli..state. +func CLIStatePath(home, configPath string) string { + name := strings.TrimSuffix(filepath.Base(configPath), filepath.Ext(configPath)) + return filepath.Join(home, fmt.Sprintf(".nvcf-cli.%s.state", name)) +} + +// CLIStateToken reads the admin token that `nvcf-cli init` stored in the +// state file at path. It fails when the file or the token is missing. +func CLIStateToken(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read nvcf-cli state: %w", err) + } + var state struct { + Token string `json:"token"` + } + if err := json.Unmarshal(data, &state); err != nil { + return "", fmt.Errorf("parse nvcf-cli state %s: %w", path, err) + } + if state.Token == "" { + return "", fmt.Errorf("nvcf-cli state %s has no admin token; run nvcf-cli init first", path) + } + return state.Token, nil +} diff --git a/tests/bdd/dsl/http_test.go b/tests/bdd/dsl/http_test.go new file mode 100644 index 0000000000..28f055c127 --- /dev/null +++ b/tests/bdd/dsl/http_test.go @@ -0,0 +1,107 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dsl + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +const ledgerEvents = `{"namespace":"ns","summary":{"total_contexts":2,"by_event":{"InstanceReady":2}},"events":[{"event_name":"InstanceReady","source":"nvidia-spot"}]}` + +func TestJSONFieldReadsNestedValues(t *testing.T) { + cases := map[string]string{ + "namespace": "ns", + "summary.total_contexts": "2", + "summary.by_event.InstanceReady": "2", + "events.0.event_name": "InstanceReady", + "events.0.source": "nvidia-spot", + } + for path, want := range cases { + got, err := JSONField(ledgerEvents, path) + if err != nil { + t.Fatalf("%s: %v", path, err) + } + if got != want { + t.Fatalf("%s = %q, want %q", path, got, want) + } + } +} + +func TestJSONFieldReturnsCompactJSONForObjects(t *testing.T) { + got, err := JSONField(ledgerEvents, "summary.by_event") + if err != nil { + t.Fatal(err) + } + if got != `{"InstanceReady":2}` { + t.Fatalf("got %q", got) + } +} + +func TestJSONFieldErrors(t *testing.T) { + for _, path := range []string{"", "missing", "events.5", "events.x", "namespace.child"} { + if _, err := JSONField(ledgerEvents, path); err == nil { + t.Fatalf("path %q: expected error", path) + } + } + if _, err := JSONField("not json", "a"); err == nil { + t.Fatal("expected parse error") + } +} + +func TestJSONFieldAtLeast(t *testing.T) { + if err := JSONFieldAtLeast(ledgerEvents, "summary.by_event.InstanceReady", 1); err != nil { + t.Fatalf("at least 1: %v", err) + } + if err := JSONFieldAtLeast(ledgerEvents, "summary.by_event.InstanceReady", 3); err == nil { + t.Fatal("expected failure for min 3") + } + if err := JSONFieldAtLeast(ledgerEvents, "namespace", 1); err == nil { + t.Fatal("expected failure for non-numeric field") + } +} + +func TestCLIStatePathMirrorsCLI(t *testing.T) { + got := CLIStatePath("/home/u", "/repo/tests/bdd/fixtures/nvcf-cli-local.yaml") + if got != "/home/u/.nvcf-cli.nvcf-cli-local.state" { + t.Fatalf("got %q", got) + } +} + +func TestCLIStateToken(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "state") + if err := os.WriteFile(path, []byte(`{"token":"abc"}`), 0o600); err != nil { + t.Fatal(err) + } + got, err := CLIStateToken(path) + if err != nil || got != "abc" { + t.Fatalf("token = %q, err = %v", got, err) + } + if err := os.WriteFile(path, []byte(`{}`), 0o600); err != nil { + t.Fatal(err) + } + if _, err := CLIStateToken(path); err == nil || !strings.Contains(err.Error(), "nvcf-cli init") { + t.Fatalf("expected missing-token error, got %v", err) + } + if _, err := CLIStateToken(filepath.Join(dir, "absent")); err == nil { + t.Fatal("expected read error") + } +} diff --git a/tests/bdd/dsl/jsonfield.go b/tests/bdd/dsl/jsonfield.go new file mode 100644 index 0000000000..45fb14d0a1 --- /dev/null +++ b/tests/bdd/dsl/jsonfield.go @@ -0,0 +1,100 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dsl + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" +) + +// JSONField reads the dotted path from a JSON document and returns the value +// as text. Object keys are matched by name and array elements by decimal +// index, so "events.0.event_name" reads the first event. Scalars come back as +// their plain text; objects and arrays come back as compact JSON. +func JSONField(raw, path string) (string, error) { + if strings.TrimSpace(path) == "" { + return "", fmt.Errorf("json path must be non-empty") + } + var node any + if err := json.Unmarshal([]byte(raw), &node); err != nil { + return "", fmt.Errorf("parse json: %w", err) + } + for _, part := range strings.Split(path, ".") { + next, err := childNode(node, part) + if err != nil { + return "", fmt.Errorf("json path %q: %w", path, err) + } + node = next + } + return nodeText(node) +} + +// JSONFieldAtLeast reports whether the numeric value at the dotted path is +// greater than or equal to min. +func JSONFieldAtLeast(raw, path string, min float64) error { + text, err := JSONField(raw, path) + if err != nil { + return err + } + value, err := strconv.ParseFloat(text, 64) + if err != nil { + return fmt.Errorf("json path %q is not a number: %q", path, text) + } + if value < min { + return fmt.Errorf("json path %q = %v, want at least %v", path, value, min) + } + return nil +} + +func childNode(node any, part string) (any, error) { + switch typed := node.(type) { + case map[string]any: + child, ok := typed[part] + if !ok { + return nil, fmt.Errorf("key %q not found", part) + } + return child, nil + case []any: + index, err := strconv.Atoi(part) + if err != nil || index < 0 || index >= len(typed) { + return nil, fmt.Errorf("index %q out of range for array of %d", part, len(typed)) + } + return typed[index], nil + default: + return nil, fmt.Errorf("cannot descend into %q: value is not an object or array", part) + } +} + +func nodeText(node any) (string, error) { + switch typed := node.(type) { + case string: + return typed, nil + case nil: + return "null", nil + case map[string]any, []any: + data, err := json.Marshal(typed) + if err != nil { + return "", fmt.Errorf("encode json: %w", err) + } + return string(data), nil + default: + return fmt.Sprint(typed), nil + } +} diff --git a/tests/bdd/features/event-ledger.feature b/tests/bdd/features/event-ledger.feature new file mode 100644 index 0000000000..894bb07a95 --- /dev/null +++ b/tests/bdd/features/event-ledger.feature @@ -0,0 +1,298 @@ +@event-ledger @ncp-local @single-cluster +Feature: Event ledger records SIS deployment events on a running self-managed stack + As a self-managed NVCF operator, + I want the event ledger to be reachable, protected, and fed by SIS, + so that I can read the lifecycle events of my function deployments. + + # This feature does not install anything. It expects the single-cluster + # ncp-local stack to be running with addons.eventLedger.enabled=true and + # nvcf-cli initialized for tests/bdd/fixtures/nvcf-cli-local.yaml. The + # NVCA OTel collector is disabled, so SIS is the only event writer. + Background: + Given these environment variables are set: + | name | + | NGC_API_KEY | + | SAMPLE_NGC_ORG | + | SAMPLE_NGC_TEAM | + | REPO_ROOT | + And I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml" + + Scenario: Event ledger is running and routed through the shared gateway + Then deployment "event-ledger" in namespace "nvcf" using context "k3d-ncp-local" should complete rollout within "2m" + And these Kubernetes resources should exist in namespace "nvcf" using context "k3d-ncp-local": + | kind | name | + | service | event-ledger | + And these Gateway API routes should be accepted and resolved using context "k3d-ncp-local" within "2m": + | kind | name | namespace | parent | + | HTTPRoute | event-ledger | envoy-gateway-system | shared-gw | + + When I successfully run command "kubectl --context k3d-ncp-local --namespace nvcf get service event-ledger -o jsonpath={.spec.ports[*].name}" + Then the command output should contain "api-port" + + When I send a "GET" request to "http://events.localhost:8080/health" + Then the HTTP status should be 200 + + Scenario: SIS is configured to send events to the event ledger + When I successfully run command "kubectl --context k3d-ncp-local --namespace sis get configmaps -o yaml" + Then the command output should contain all: + | text | + | ICMS_FNDS_MESSAGES_ENABLED: "true" | + | ICMS_FNDS_MESSAGES_V3_ENABLED: "true" | + + # A missing or JWT-shaped invalid credential is rejected with 401. An opaque + # token is not a JWT, so it goes to the api-keys policy evaluator, which + # denies an unknown key with 403. + Scenario: Requests without valid credentials are rejected + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/cloudevents" with body: + """ + {"specversion":"1.0"} + """ + Then the HTTP status should be 401 + + Given the HTTP request header "Authorization" is "Bearer eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ4In0.c2ln" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/cloudevents" + Then the HTTP status should be 401 + + Given the HTTP request header "Authorization" is "Bearer not-a-token" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/k8s-events" + Then the HTTP status should be 403 + + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-auth/stats" + Then the HTTP status should be 401 + + Given the HTTP request header "Authorization" is "Bearer eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ4In0.c2ln" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-auth/events" + Then the HTTP status should be 401 + + Given the HTTP request header "Authorization" is "Bearer nvapi-not-a-real-key" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-auth/events" + Then the HTTP status should be 403 + + When I send a "GET" request to "http://events.localhost:8080/status" + Then the HTTP status should be 401 + + Scenario: V1 and V2 endpoints are disabled + When I send a "POST" request to "http://events.localhost:8080/v1/ledger/versions/bdd-version/instances/bdd-instance" with body: + """ + {} + """ + Then the HTTP status should be 404 + + When I send a "POST" request to "http://events.localhost:8080/v2/ledger/versions/bdd-version/deployments/bdd-deployment/instances/bdd-instance" with body: + """ + {} + """ + Then the HTTP status should be 404 + + Scenario: A read key can query the ledger and bad queries are rejected + When I successfully run command: + """ + /bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z' + """ + And I export command output to environment variable "BDD_EL_KEY_EXPIRES" + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "POST" request to "http://api-keys.localhost:8080/v1/keys" with body: + """ + { + "description": "bdd-event-ledger-read", + "expires_at": "${BDD_EL_KEY_EXPIRES}", + "authorizations": { + "policies": [ + { + "aud": "nvidia-event-ledger-ncp-service-id-ckozoh6f", + "auds": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"], + "product": "nv-cloud-functions", + "resources": [{"id": "*", "type": "account-functions"}], + "scopes": ["fnds:getEvents", "fnds:getStats"] + } + ] + }, + "audience_service_ids": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"] + } + """ + Then the HTTP status should be 200 + When I export the HTTP response JSON field "value" to environment variable "BDD_EL_READ_KEY" + And I export the HTTP response JSON field "id" to environment variable "BDD_EL_READ_KEY_ID" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/stats" + Then the HTTP status should be 200 + And the HTTP response JSON field "namespace" should equal "bdd-no-such-namespace" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/events" + Then the HTTP status should be 200 + + # Malformed queries must get a 400 before reaching Cassandra: a half-supplied + # attribute filter, a disallowed character in a context id, and an unknown stats view. + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/events?attribute_key=only-a-key" + Then the HTTP status should be 400 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/events?instance_id=bad!id" + Then the HTTP status should be 400 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/stats?view=bogus" + Then the HTTP status should be 400 + + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" + Then the HTTP status should be 204 + + # Design: write access needs an OpenBao JWT with the FnDS write scope. A + # read-scoped api key must not be able to write. Tagged because this + # currently returns 200 and stores the event; the scenario writes data, so + # the read-only live entry point excludes it. + @known-defect + Scenario: A read-scoped key cannot write events + When I successfully run command: + """ + /bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z' + """ + And I export command output to environment variable "BDD_EL_KEY_EXPIRES" + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "POST" request to "http://api-keys.localhost:8080/v1/keys" with body: + """ + { + "description": "bdd-event-ledger-read-only-write", + "expires_at": "${BDD_EL_KEY_EXPIRES}", + "authorizations": { + "policies": [ + { + "aud": "nvidia-event-ledger-ncp-service-id-ckozoh6f", + "auds": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"], + "product": "nv-cloud-functions", + "resources": [{"id": "*", "type": "account-functions"}], + "scopes": ["fnds:getEvents", "fnds:getStats"] + } + ] + }, + "audience_service_ids": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"] + } + """ + Then the HTTP status should be 200 + When I export the HTTP response JSON field "value" to environment variable "BDD_EL_READ_KEY" + And I export the HTTP response JSON field "id" to environment variable "BDD_EL_READ_KEY_ID" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + And the HTTP request header "Content-Type" is "application/cloudevents+json" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/cloudevents" with body: + """ + {"specversion":"1.0","id":"bdd-write-attempt-1","type":"BddWriteAttempt","source":"bdd","namespace":"bdd-write-attempt"} + """ + Then the HTTP status should be 401 or 403 + + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" + Then the HTTP status should be 204 + + @function-lifecycle + Scenario: A function deployment produces SIS events that can be read back + When I successfully run command: + """ + /bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z' + """ + And I export command output to environment variable "BDD_EL_KEY_EXPIRES" + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "POST" request to "http://api-keys.localhost:8080/v1/keys" with body: + """ + { + "description": "bdd-event-ledger-lifecycle", + "expires_at": "${BDD_EL_KEY_EXPIRES}", + "authorizations": { + "policies": [ + { + "aud": "nvidia-event-ledger-ncp-service-id-ckozoh6f", + "auds": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"], + "product": "nv-cloud-functions", + "resources": [{"id": "*", "type": "account-functions"}], + "scopes": ["fnds:getEvents", "fnds:getStats"] + } + ] + }, + "audience_service_ids": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"] + } + """ + Then the HTTP status should be 200 + When I export the HTTP response JSON field "value" to environment variable "BDD_EL_READ_KEY" + And I export the HTTP response JSON field "id" to environment variable "BDD_EL_READ_KEY_ID" + + When I successfully create function "bdd-event-ledger" from image "nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/load_tester_supreme:0.0.8" with CLI options: + | option | value | + | --inference-url | /echo | + | --inference-port | 8000 | + | --health-uri | /health | + | --health-port | 8000 | + | --health-timeout | PT30S | + And I successfully deploy the function selected by NVCF CLI with options: + | option | value | + | --gpu | H100 | + | --instance-type | NCP.GPU.H100_1x | + | --backend | ncp-local | + | --regions | us-west-1 | + | --min-instances | 1 | + | --max-instances | 1 | + | --timeout | 900 | + And I export the function selected by NVCF CLI to environment variables "BDD_EL_FUNCTION_ID" and "BDD_EL_VERSION_ID" + + # SIS writes InstanceReady once the instance is running. + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I poll "GET" "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/events" for up to 120 seconds until the HTTP response body contains "InstanceReady" + Then the HTTP status should be 200 + And the HTTP response body should contain "nvidia-spot" + And the HTTP response body should not contain "InstanceDestroyed" + # The NVCA collector is disabled, so nothing else writes to this namespace. + And the HTTP response body should not contain "ICMSRequest." + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats" + Then the HTTP status should be 200 + And the HTTP response JSON field "namespace" should equal "${BDD_EL_VERSION_ID}" + And the HTTP response JSON field "summary.by_event.InstanceReady" should be at least 1 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats?eventFilter=InstanceReady" + Then the HTTP status should be 200 + And the HTTP response body should contain "InstanceReady" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats?eventFilter=NoSuchEvent" + Then the HTTP status should be 200 + And the HTTP response body should not contain "InstanceReady" + + # Undeploy removes the instance; SIS then reports it destroyed. + When I successfully undeploy the function selected by NVCF CLI + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I poll "GET" "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/events" for up to 180 seconds until the HTTP response body contains "InstanceDestroyed" + Then the HTTP status should be 200 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats" + Then the HTTP response JSON field "summary.by_event.InstanceDestroyed" should be at least 1 + + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" + Then the HTTP status should be 204 diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 9e2d61ca78..bf8e926578 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -2663,6 +2663,27 @@ func TestObservabilityDisabled(t *testing.T) { runLiveFeature(t, "observability-disabled.feature") } +// TestEventLedger is the live entry point for the event-ledger feature. It +// runs against an already-running single-cluster ncp-local stack and installs +// nothing. Skipped under -short. +func TestEventLedger(t *testing.T) { + if testing.Short() { + t.Skip("live run skipped under -short") + } + runLiveFeature(t, "event-ledger.feature") +} + +// TestEventLedgerReadOnly runs the event-ledger feature without the function +// lifecycle scenario (no GPU capacity or image pull access needed) and without +// the known-defect scenario (it writes an event). +// Skipped under -short. +func TestEventLedgerReadOnly(t *testing.T) { + if testing.Short() { + t.Skip("live run skipped under -short") + } + runLiveFeatureTags(t, "event-ledger.feature", "~@function-lifecycle && ~@known-defect") +} + // TestMultiClusterHelmfile is the live entry point for the // multi-cluster Helmfile feature: control-plane install on // k3d-ncp-local-cp followed by register + NVCA install on the @@ -2888,3 +2909,140 @@ func mustResolveFeaturePath(t *testing.T, name string) string { t.Helper() return filepath.Join("features", name) } + +// eventLedgerHTTPFake answers the event-ledger feature's HTTP calls the way +// the real services are expected to, and records requests for assertions. +type eventLedgerHTTPFake struct { + destroyed *bool + requests []string +} + +func (f *eventLedgerHTTPFake) Do(_ context.Context, method, rawURL string, headers map[string]string, _ string) (harness.HTTPResponse, error) { + f.requests = append(f.requests, method+" "+rawURL) + hostAndPath := strings.TrimPrefix(rawURL, "http://") + host, rest, _ := strings.Cut(hostAndPath, "/") + path, query, _ := strings.Cut("/"+rest, "?") + auth := headers["Authorization"] + admin := auth == "Bearer admin-jwt" + reader := auth == "Bearer nvapi-test" + + if strings.HasPrefix(host, "api-keys.") { + switch { + case !admin || headers["Key-Issuer-Id"] == "": + return harness.HTTPResponse{Status: 401}, nil + case method == "POST": + return harness.HTTPResponse{Status: 200, Body: `{"id":"key-1","value":"nvapi-test"}`}, nil + default: + return harness.HTTPResponse{Status: 204}, nil + } + } + switch { + case path == "/health": + return harness.HTTPResponse{Status: 200}, nil + case strings.HasPrefix(path, "/v1/"), strings.HasPrefix(path, "/v2/"): + return harness.HTTPResponse{Status: 404}, nil + case auth == "", strings.HasPrefix(auth, "Bearer eyJ"): + return harness.HTTPResponse{Status: 401}, nil + case !reader: + return harness.HTTPResponse{Status: 403}, nil + case method == "POST": + return harness.HTTPResponse{Status: 403}, nil + } + namespace := strings.Split(path, "/")[4] + switch { + case strings.Contains(query, "attribute_key"), strings.Contains(query, "instance_id=bad"), strings.Contains(query, "view=bogus"): + return harness.HTTPResponse{Status: 400}, nil + case strings.HasSuffix(path, "/stats"): + if strings.Contains(query, "NoSuchEvent") { + return harness.HTTPResponse{Status: 200, Body: `{"namespace":"` + namespace + `","contexts":[]}`}, nil + } + return harness.HTTPResponse{Status: 200, Body: `{"namespace":"` + namespace + `","summary":{"by_event":{"InstanceReady":1,"InstanceDestroyed":1}}}`}, nil + case namespace == "version-1": + body := `{"namespace":"version-1","events":[{"event_name":"InstanceReady","source":"nvidia-spot"}` + if *f.destroyed { + body += `,{"event_name":"InstanceDestroyed","source":"nvidia-spot"}` + } + return harness.HTTPResponse{Status: 200, Body: body + `]}`}, nil + default: + return harness.HTTPResponse{Status: 200, Body: `{"namespace":"` + namespace + `","events":[]}`}, nil + } +} + +func TestEventLedgerFeatureFileWiresToSteps(t *testing.T) { + const selectedFunctionStatusCommand = `/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml status --json` + home := t.TempDir() + t.Setenv("HOME", home) + if err := os.WriteFile(filepath.Join(home, ".nvcf-cli.nvcf-cli-local.state"), []byte(`{"token":"admin-jwt"}`), 0o600); err != nil { + t.Fatalf("seed cli state: %v", err) + } + t.Setenv("NGC_API_KEY", "test-key") + t.Setenv("SAMPLE_NGC_ORG", "test-org") + t.Setenv("SAMPLE_NGC_TEAM", "test-team") + t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") + t.Setenv("REPO_ROOT", "/repo-root-placeholder") + destroyed := false + runner := newFakeRunner(map[string]harness.Result{ + selectedFunctionStatusCommand: { + ExitCode: 0, + Stdout: `{"currentFunction":{"hasFunction":true,"functionId":"function-1","versionId":"version-1"}}`, + }, + `/bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z'`: {ExitCode: 0, Stdout: "2030-01-01T00:00:00.000Z\n"}, + "kubectl --context k3d-ncp-local --namespace nvcf get service event-ledger -o jsonpath={.spec.ports[*].name}": { + ExitCode: 0, + Stdout: "api-port metrics", + }, + "kubectl --context k3d-ncp-local --namespace sis get configmaps -o yaml": { + ExitCode: 0, + Stdout: " ICMS_FNDS_MESSAGES_ENABLED: \"true\"\n ICMS_FNDS_MESSAGES_V3_ENABLED: \"true\"\n", + }, + }) + runner.onRun = func(command string) error { + if strings.Contains(command, " function delete --deployment-only") { + destroyed = true + } + return nil + } + suite := newWiringSuite(t, runner) + httpFake := &eventLedgerHTTPFake{destroyed: &destroyed} + suite.HTTP = httpFake + + sc := steps.NewScenarioContext(suite) + featurePath := mustResolveFeaturePath(t, "event-ledger.feature") + var out strings.Builder + status := godog.TestSuite{ + Name: "event-ledger-wiring", + ScenarioInitializer: func(ctx *godog.ScenarioContext) { + steps.RegisterAll(ctx, sc) + }, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{featurePath}, + Strict: true, + Output: &out, + }, + }.Run() + if status != 0 { + t.Fatalf("godog suite status = %d\n%s", status, out.String()) + } + runs := suite.Runner.(*fakeRunner).runs + if !commandRanThatContains(runs, " function deploy create ") { + t.Fatal("function deploy command was never invoked") + } + if !commandRanThatContains(runs, " function delete --deployment-only") { + t.Fatal("function undeploy command was never invoked") + } + for _, secret := range []string{"admin-jwt", "nvapi-test"} { + if commandRanThatContains(runs, secret) { + t.Fatalf("secret %q leaked into a command line", secret) + } + } + deletes := 0 + for _, request := range httpFake.requests { + if request == "DELETE http://api-keys.localhost:8080/v1/keys/key-1" { + deletes++ + } + } + if deletes != 3 { + t.Fatalf("read keys deleted = %d, want 3", deletes) + } +} diff --git a/tests/bdd/harness/http.go b/tests/bdd/harness/http.go new file mode 100644 index 0000000000..449d67b379 --- /dev/null +++ b/tests/bdd/harness/http.go @@ -0,0 +1,82 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package harness + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +const httpTimeout = 30 * time.Second + +// HTTPResponse is the status code and body of one HTTP exchange. +type HTTPResponse struct { + Status int + Body string +} + +// HTTPClient sends one HTTP request. Header values may carry credentials, +// so implementations must not log them. +type HTTPClient interface { + Do(ctx context.Context, method, url string, headers map[string]string, body string) (HTTPResponse, error) +} + +type netHTTPClient struct { + client *http.Client +} + +// NewHTTPClient returns an HTTPClient backed by net/http. Redirects are not +// followed so a test can assert on a 301 directly. +func NewHTTPClient() HTTPClient { + return &netHTTPClient{client: &http.Client{ + Timeout: httpTimeout, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }} +} + +// Do sends the request and returns the status and full body. The error text +// never includes header values. +func (c *netHTTPClient) Do(ctx context.Context, method, url string, headers map[string]string, body string) (HTTPResponse, error) { + var reader io.Reader + if body != "" { + reader = strings.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, url, reader) + if err != nil { + return HTTPResponse{}, fmt.Errorf("build %s request to %s: %w", method, url, err) + } + for name, value := range headers { + req.Header.Set(name, value) + } + resp, err := c.client.Do(req) + if err != nil { + return HTTPResponse{}, fmt.Errorf("%s %s: %w", method, url, err) + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + return HTTPResponse{}, fmt.Errorf("read %s %s response: %w", method, url, err) + } + return HTTPResponse{Status: resp.StatusCode, Body: string(data)}, nil +} diff --git a/tests/bdd/harness/http_test.go b/tests/bdd/harness/http_test.go new file mode 100644 index 0000000000..1ca65ce2b0 --- /dev/null +++ b/tests/bdd/harness/http_test.go @@ -0,0 +1,75 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package harness + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestNetHTTPClientSendsHeadersAndBody(t *testing.T) { + var gotAuth, gotBody, gotMethod string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + gotMethod = r.Method + data, _ := io.ReadAll(r.Body) + gotBody = string(data) + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + defer server.Close() + + resp, err := NewHTTPClient().Do(context.Background(), "POST", server.URL, map[string]string{"Authorization": "Bearer t"}, `{"a":1}`) + if err != nil { + t.Fatal(err) + } + if resp.Status != http.StatusCreated || resp.Body != `{"ok":true}` { + t.Fatalf("resp = %+v", resp) + } + if gotAuth != "Bearer t" || gotBody != `{"a":1}` || gotMethod != "POST" { + t.Fatalf("server saw auth=%q body=%q method=%q", gotAuth, gotBody, gotMethod) + } +} + +func TestNetHTTPClientDoesNotFollowRedirects(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "/elsewhere", http.StatusMovedPermanently) + })) + defer server.Close() + resp, err := NewHTTPClient().Do(context.Background(), "GET", server.URL, nil, "") + if err != nil { + t.Fatal(err) + } + if resp.Status != http.StatusMovedPermanently { + t.Fatalf("status = %d, want 301", resp.Status) + } +} + +func TestNetHTTPClientErrorOmitsHeaderValues(t *testing.T) { + _, err := NewHTTPClient().Do(context.Background(), "GET", "http://127.0.0.1:1", map[string]string{"Authorization": "Bearer secret-token"}, "") + if err == nil { + t.Fatal("expected connection error") + } + if strings.Contains(err.Error(), "secret-token") { + t.Fatalf("error leaks header value: %v", err) + } +} diff --git a/tests/bdd/harness/suite.go b/tests/bdd/harness/suite.go index 9094c685eb..ff967ae351 100644 --- a/tests/bdd/harness/suite.go +++ b/tests/bdd/harness/suite.go @@ -40,6 +40,7 @@ type Suite struct { Ledger *Ledger EnvLedger *EnvLedger Cache *CommandCache + HTTP HTTPClient signalMu sync.Mutex signalContext context.Context diff --git a/tests/bdd/steps/context.go b/tests/bdd/steps/context.go index a5683a7bd6..4dc723be70 100644 --- a/tests/bdd/steps/context.go +++ b/tests/bdd/steps/context.go @@ -54,6 +54,8 @@ type ScenarioContext struct { LastCommand string NVCFCLIConfig string Manifests map[string]string + HTTPHeaders map[string]string + LastHTTP *harness.HTTPResponse } // NewScenarioContext wraps suite in a fresh per-scenario state. The @@ -72,6 +74,8 @@ func RegisterAll(ctx *godog.ScenarioContext, sc *ScenarioContext) { sc.LastCommand = "" sc.NVCFCLIConfig = "" sc.Manifests = nil + sc.HTTPHeaders = nil + sc.LastHTTP = nil return c, nil }) // Godog's default pretty formatter buffers the scenario block until @@ -101,6 +105,7 @@ func RegisterAll(ctx *godog.ScenarioContext, sc *ScenarioContext) { registerFileSteps(ctx, sc) registerManifestSteps(ctx, sc) registerCommandSteps(ctx, sc) + registerHTTPSteps(ctx, sc) registerNVCFCLISteps(ctx, sc) registerRegistrationSteps(ctx, sc) registerAssertionSteps(ctx, sc) diff --git a/tests/bdd/steps/http_steps.go b/tests/bdd/steps/http_steps.go new file mode 100644 index 0000000000..c135ad15dd --- /dev/null +++ b/tests/bdd/steps/http_steps.go @@ -0,0 +1,264 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package steps + +import ( + "context" + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/cucumber/godog" + + "nvcf-bdd/dsl" + "nvcf-bdd/harness" +) + +const maxHTTPFailureBody = 300 + +// httpPollInterval is the pause between polls; tests shorten it. +var httpPollInterval = 5 * time.Second + +// registerHTTPSteps hooks the HTTP request and response assertion steps. +// Request headers are declared with their own steps and apply to the next +// request only, so a header value never appears in the send step text. +func registerHTTPSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { + ctx.Step(`^the HTTP request header "([^"]*)" is "([^"]*)"$`, sc.theHTTPRequestHeaderIs) + ctx.Step(`^the HTTP request header "([^"]*)" is the NVCF CLI admin token with prefix "([^"]*)"$`, sc.theHTTPRequestHeaderIsAdminToken) + ctx.Step(`^I send an? "([A-Z]+)" request to "([^"]*)"$`, sc.iSendHTTPRequest) + ctx.Step(`^I send an? "([A-Z]+)" request to "([^"]*)" with body:$`, sc.iSendHTTPRequestWithBody) + ctx.Step(`^I poll "([A-Z]+)" "([^"]*)" for up to (\d+) seconds until the HTTP response body contains "([^"]*)"$`, sc.iPollHTTPUntilBodyContains) + ctx.Step(`^the HTTP status should be (\d+)$`, sc.theHTTPStatusShouldBe) + ctx.Step(`^the HTTP status should be (\d+) or (\d+)$`, sc.theHTTPStatusShouldBeOneOf) + ctx.Step(`^the HTTP response body should contain "([^"]*)"$`, sc.theHTTPBodyShouldContain) + ctx.Step(`^the HTTP response body should not contain "([^"]*)"$`, sc.theHTTPBodyShouldNotContain) + ctx.Step(`^the HTTP response JSON field "([^"]*)" should equal "([^"]*)"$`, sc.theHTTPJSONFieldShouldEqual) + ctx.Step(`^the HTTP response JSON field "([^"]*)" should be at least (\d+)$`, sc.theHTTPJSONFieldShouldBeAtLeast) + ctx.Step(`^I export the HTTP response JSON field "([^"]*)" to environment variable "([^"]*)"$`, sc.iExportHTTPJSONField) +} + +func (sc *ScenarioContext) httpClient() harness.HTTPClient { + if sc.Suite.HTTP == nil { + sc.Suite.HTTP = harness.NewHTTPClient() + } + return sc.Suite.HTTP +} + +func (sc *ScenarioContext) theHTTPRequestHeaderIs(name, value string) error { + if strings.TrimSpace(name) == "" { + return fmt.Errorf("http header name must be non-empty") + } + if sc.HTTPHeaders == nil { + sc.HTTPHeaders = map[string]string{} + } + sc.HTTPHeaders[name] = dsl.Interpolate(value) + return nil +} + +// theHTTPRequestHeaderIsAdminToken reads the admin token from the state file +// of the selected NVCF CLI config so the JWT never enters argv, env or logs. +func (sc *ScenarioContext) theHTTPRequestHeaderIsAdminToken(name, prefix string) error { + if sc.NVCFCLIConfig == "" { + return fmt.Errorf("no NVCF CLI config selected; use 'I use NVCF CLI config' first") + } + home, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("resolve home dir: %w", err) + } + token, err := dsl.CLIStateToken(dsl.CLIStatePath(home, sc.NVCFCLIConfig)) + if err != nil { + return err + } + if sc.HTTPHeaders == nil { + sc.HTTPHeaders = map[string]string{} + } + sc.HTTPHeaders[name] = prefix + token + return nil +} + +func (sc *ScenarioContext) iSendHTTPRequest(ctx context.Context, method, url string) error { + return sc.sendHTTP(ctx, method, url, "") +} + +func (sc *ScenarioContext) iSendHTTPRequestWithBody(ctx context.Context, method, url string, doc *godog.DocString) error { + return sc.sendHTTP(ctx, method, url, doc.Content) +} + +// sendHTTP interpolates the URL and body, sends one request with the pending +// headers, and records the response. A non-2xx status is not a step failure; +// the status assertion decides. +func (sc *ScenarioContext) sendHTTP(ctx context.Context, method, url, body string) error { + headers := sc.HTTPHeaders + sc.HTTPHeaders = nil + resp, err := sc.httpClient().Do(ctx, method, dsl.Interpolate(url), headers, dsl.Interpolate(body)) + if err != nil { + sc.LastHTTP = nil + return err + } + sc.LastHTTP = &resp + return nil +} + +func (sc *ScenarioContext) iPollHTTPUntilBodyContains(ctx context.Context, method, url string, seconds int, text string) error { + want := dsl.Interpolate(text) + if strings.TrimSpace(want) == "" { + return fmt.Errorf("poll text must be non-empty") + } + headers := sc.HTTPHeaders + sc.HTTPHeaders = nil + deadline := time.Now().Add(time.Duration(seconds) * time.Second) + for { + resp, err := sc.httpClient().Do(ctx, method, dsl.Interpolate(url), headers, "") + if err == nil { + sc.LastHTTP = &resp + if strings.Contains(resp.Body, want) { + return nil + } + } + if time.Now().Add(httpPollInterval).After(deadline) { + if err != nil { + return fmt.Errorf("poll timed out after %d seconds: %w", seconds, err) + } + return fmt.Errorf("poll timed out after %d seconds: last status %d, body missing %q: %s", + seconds, sc.LastHTTP.Status, want, truncateBody(sc.LastHTTP.Body)) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(httpPollInterval): + } + } +} + +func (sc *ScenarioContext) requireHTTP() (harness.HTTPResponse, error) { + if sc.LastHTTP == nil { + return harness.HTTPResponse{}, fmt.Errorf("no HTTP response recorded; send a request first") + } + return *sc.LastHTTP, nil +} + +func (sc *ScenarioContext) theHTTPStatusShouldBe(want int) error { + resp, err := sc.requireHTTP() + if err != nil { + return err + } + if resp.Status != want { + return fmt.Errorf("http status = %d, want %d: %s", resp.Status, want, truncateBody(resp.Body)) + } + return nil +} + +func (sc *ScenarioContext) theHTTPStatusShouldBeOneOf(first, second int) error { + resp, err := sc.requireHTTP() + if err != nil { + return err + } + if resp.Status != first && resp.Status != second { + return fmt.Errorf("http status = %d, want %d or %d: %s", resp.Status, first, second, truncateBody(resp.Body)) + } + return nil +} + +func (sc *ScenarioContext) theHTTPBodyShouldContain(text string) error { + resp, err := sc.requireHTTP() + if err != nil { + return err + } + want := dsl.Interpolate(text) + if strings.TrimSpace(want) == "" { + return fmt.Errorf("expected text must be non-empty") + } + if !strings.Contains(resp.Body, want) { + return fmt.Errorf("http response body missing %q: %s", want, truncateBody(resp.Body)) + } + return nil +} + +func (sc *ScenarioContext) theHTTPBodyShouldNotContain(text string) error { + resp, err := sc.requireHTTP() + if err != nil { + return err + } + unwanted := dsl.Interpolate(text) + if strings.TrimSpace(unwanted) == "" { + return fmt.Errorf("unwanted text must be non-empty") + } + if strings.Contains(resp.Body, unwanted) { + return fmt.Errorf("http response body unexpectedly contains %q", unwanted) + } + return nil +} + +func (sc *ScenarioContext) theHTTPJSONFieldShouldEqual(path, want string) error { + resp, err := sc.requireHTTP() + if err != nil { + return err + } + got, err := dsl.JSONField(resp.Body, path) + if err != nil { + return err + } + if expected := dsl.Interpolate(want); got != expected { + return fmt.Errorf("json field %q = %q, want %q", path, got, expected) + } + return nil +} + +func (sc *ScenarioContext) theHTTPJSONFieldShouldBeAtLeast(path string, min int) error { + resp, err := sc.requireHTTP() + if err != nil { + return err + } + return dsl.JSONFieldAtLeast(resp.Body, path, float64(min)) +} + +// iExportHTTPJSONField stores a response field in an env var through the +// EnvLedger, which restores the prior value at suite teardown. The value is +// never echoed. +func (sc *ScenarioContext) iExportHTTPJSONField(path, name string) error { + if name == "" { + return fmt.Errorf("export: env var name must be non-empty") + } + resp, err := sc.requireHTTP() + if err != nil { + return err + } + value, err := dsl.JSONField(resp.Body, path) + if err != nil { + return err + } + if strings.TrimSpace(value) == "" { + return fmt.Errorf("export to %s: json field %q is empty", name, path) + } + if err := sc.Suite.EnvLedger.Snapshot(name); err != nil { + return fmt.Errorf("export to %s: snapshot: %w", name, err) + } + if err := os.Setenv(name, value); err != nil { + return fmt.Errorf("export to %s: setenv: %w", name, err) + } + return nil +} + +func truncateBody(body string) string { + if len(body) <= maxHTTPFailureBody { + return body + } + return body[:maxHTTPFailureBody] + "..." + strconv.Itoa(len(body)) + " bytes" +} diff --git a/tests/bdd/steps/http_steps_test.go b/tests/bdd/steps/http_steps_test.go new file mode 100644 index 0000000000..79469012b7 --- /dev/null +++ b/tests/bdd/steps/http_steps_test.go @@ -0,0 +1,224 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package steps + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "nvcf-bdd/harness" +) + +type fakeHTTP struct { + responses []harness.HTTPResponse + err error + calls int + method string + url string + headers map[string]string + body string +} + +func (f *fakeHTTP) Do(_ context.Context, method, url string, headers map[string]string, body string) (harness.HTTPResponse, error) { + f.method, f.url, f.headers, f.body = method, url, headers, body + i := f.calls + f.calls++ + if f.err != nil { + return harness.HTTPResponse{}, f.err + } + if i >= len(f.responses) { + i = len(f.responses) - 1 + } + return f.responses[i], nil +} + +func newHTTPScenario(t *testing.T, responses ...harness.HTTPResponse) (*ScenarioContext, *fakeHTTP) { + t.Helper() + sc, _ := newScenarioContext(t) + fake := &fakeHTTP{responses: responses} + sc.Suite.HTTP = fake + return sc, fake +} + +func TestSendHTTPAppliesPendingHeadersOnceAndInterpolates(t *testing.T) { + t.Setenv("BDD_HTTP_HOST", "events.localhost:8080") + sc, fake := newHTTPScenario(t, harness.HTTPResponse{Status: 200, Body: "{}"}) + if err := sc.theHTTPRequestHeaderIs("Authorization", "Bearer ${BDD_HTTP_HOST}"); err != nil { + t.Fatal(err) + } + if err := sc.iSendHTTPRequest(context.Background(), "GET", "http://${BDD_HTTP_HOST}/health"); err != nil { + t.Fatal(err) + } + if fake.url != "http://events.localhost:8080/health" || fake.headers["Authorization"] != "Bearer events.localhost:8080" { + t.Fatalf("url=%q headers=%v", fake.url, fake.headers) + } + if err := sc.iSendHTTPRequest(context.Background(), "GET", "http://x/"); err != nil { + t.Fatal(err) + } + if len(fake.headers) != 0 { + t.Fatalf("headers leaked to next request: %v", fake.headers) + } +} + +func TestHTTPStatusAssertion(t *testing.T) { + sc, _ := newHTTPScenario(t, harness.HTTPResponse{Status: 401, Body: "nope"}) + if err := sc.theHTTPStatusShouldBe(401); err == nil { + t.Fatal("assertion must fail before any request") + } + if err := sc.iSendHTTPRequest(context.Background(), "POST", "http://x/"); err != nil { + t.Fatal(err) + } + if err := sc.theHTTPStatusShouldBe(401); err != nil { + t.Fatalf("401: %v", err) + } + if err := sc.theHTTPStatusShouldBe(200); err == nil || !strings.Contains(err.Error(), "nope") { + t.Fatalf("expected status mismatch with body, got %v", err) + } +} + +func TestHTTPStatusOneOfAssertion(t *testing.T) { + sc, _ := newHTTPScenario(t, harness.HTTPResponse{Status: 403, Body: "forbidden"}) + if err := sc.iSendHTTPRequest(context.Background(), "POST", "http://x/"); err != nil { + t.Fatal(err) + } + if err := sc.theHTTPStatusShouldBeOneOf(401, 403); err != nil { + t.Fatalf("403 should satisfy 401 or 403: %v", err) + } + if err := sc.theHTTPStatusShouldBeOneOf(200, 201); err == nil { + t.Fatal("expected mismatch") + } +} + +func TestHTTPSendErrorClearsLastResponse(t *testing.T) { + sc, fake := newHTTPScenario(t, harness.HTTPResponse{Status: 200}) + if err := sc.iSendHTTPRequest(context.Background(), "GET", "http://x/"); err != nil { + t.Fatal(err) + } + fake.err = errors.New("connection refused") + if err := sc.iSendHTTPRequest(context.Background(), "GET", "http://x/"); err == nil { + t.Fatal("expected send error") + } + if err := sc.theHTTPStatusShouldBe(200); err == nil { + t.Fatal("stale response must not satisfy assertion") + } +} + +func TestHTTPBodyAndJSONAssertions(t *testing.T) { + sc, _ := newHTTPScenario(t, harness.HTTPResponse{Status: 200, Body: `{"summary":{"by_event":{"InstanceReady":2}},"events":[{"source":"nvidia-spot"}]}`}) + if err := sc.iSendHTTPRequest(context.Background(), "GET", "http://x/"); err != nil { + t.Fatal(err) + } + if err := sc.theHTTPBodyShouldContain("InstanceReady"); err != nil { + t.Fatal(err) + } + if err := sc.theHTTPBodyShouldNotContain("InstanceDestroyed"); err != nil { + t.Fatal(err) + } + if err := sc.theHTTPBodyShouldNotContain("InstanceReady"); err == nil { + t.Fatal("not-contain must fail when present") + } + if err := sc.theHTTPBodyShouldContain(" "); err == nil { + t.Fatal("blank text must be rejected") + } + if err := sc.theHTTPJSONFieldShouldEqual("events.0.source", "nvidia-spot"); err != nil { + t.Fatal(err) + } + if err := sc.theHTTPJSONFieldShouldEqual("events.0.source", "nvca"); err == nil { + t.Fatal("expected mismatch") + } + if err := sc.theHTTPJSONFieldShouldBeAtLeast("summary.by_event.InstanceReady", 1); err != nil { + t.Fatal(err) + } +} + +func TestExportHTTPJSONFieldSetsEnvAndRestores(t *testing.T) { + sc, _ := newHTTPScenario(t, harness.HTTPResponse{Status: 200, Body: `{"value":"nvapi-abc"}`}) + t.Setenv("BDD_EXPORT_KEY", "before") + if err := sc.iSendHTTPRequest(context.Background(), "POST", "http://x/"); err != nil { + t.Fatal(err) + } + if err := sc.iExportHTTPJSONField("value", "BDD_EXPORT_KEY"); err != nil { + t.Fatal(err) + } + if got := os.Getenv("BDD_EXPORT_KEY"); got != "nvapi-abc" { + t.Fatalf("env = %q", got) + } + if err := sc.Suite.EnvLedger.RestoreAll(); err != nil { + t.Fatal(err) + } + if got := os.Getenv("BDD_EXPORT_KEY"); got != "before" { + t.Fatalf("env after restore = %q", got) + } + if err := sc.iExportHTTPJSONField("missing", "BDD_EXPORT_KEY"); err == nil { + t.Fatal("missing field must fail") + } +} + +func TestPollHTTPRetriesUntilBodyContains(t *testing.T) { + old := httpPollInterval + httpPollInterval = time.Millisecond + t.Cleanup(func() { httpPollInterval = old }) + sc, fake := newHTTPScenario(t, + harness.HTTPResponse{Status: 200, Body: `{"events":[]}`}, + harness.HTTPResponse{Status: 200, Body: `{"events":[{"event_name":"InstanceReady"}]}`}, + ) + if err := sc.iPollHTTPUntilBodyContains(context.Background(), "GET", "http://x/", 5, "InstanceReady"); err != nil { + t.Fatal(err) + } + if fake.calls != 2 { + t.Fatalf("calls = %d, want 2", fake.calls) + } +} + +func TestPollHTTPTimesOut(t *testing.T) { + old := httpPollInterval + httpPollInterval = 400 * time.Millisecond + t.Cleanup(func() { httpPollInterval = old }) + sc, _ := newHTTPScenario(t, harness.HTTPResponse{Status: 200, Body: `{"events":[]}`}) + err := sc.iPollHTTPUntilBodyContains(context.Background(), "GET", "http://x/", 1, "InstanceReady") + if err == nil || !strings.Contains(err.Error(), "timed out") { + t.Fatalf("expected timeout, got %v", err) + } +} + +func TestAdminTokenHeaderReadsStateFile(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + if err := os.WriteFile(filepath.Join(home, ".nvcf-cli.nvcf-cli-local.state"), []byte(`{"token":"jwt-1"}`), 0o600); err != nil { + t.Fatal(err) + } + sc, fake := newHTTPScenario(t, harness.HTTPResponse{Status: 200}) + if err := sc.theHTTPRequestHeaderIsAdminToken("Authorization", "Bearer "); err == nil { + t.Fatal("must require a selected CLI config") + } + sc.NVCFCLIConfig = "/repo/tests/bdd/fixtures/nvcf-cli-local.yaml" + if err := sc.theHTTPRequestHeaderIsAdminToken("Authorization", "Bearer "); err != nil { + t.Fatal(err) + } + if err := sc.iSendHTTPRequest(context.Background(), "GET", "http://x/"); err != nil { + t.Fatal(err) + } + if fake.headers["Authorization"] != "Bearer jwt-1" { + t.Fatalf("headers = %v", fake.headers) + } +} From 0b324c078122b1a2256bb1072ec9b679e49997ab Mon Sep 17 00:00:00 2001 From: Shelley Shen Date: Sat, 10 Oct 2026 22:28:58 -0700 Subject: [PATCH 02/11] test(bdd): read event-ledger events by context in the lifecycle scenario Event-ledger stores events per (namespace, context). Reading a namespace's events without a context only returns context-less events, so the lifecycle scenario never saw the events SIS writes for a deployment. Poll stats, which list every context in the namespace, then read events with the context stats reports. Wait up to five minutes for the instance to reach running, and keep the wiring test's fake server in step with the context-based reads. Signed-off-by: Shelley Shen --- tests/bdd/features/event-ledger.feature | 38 +++++++++++++++++-------- tests/bdd/godog_test.go | 20 +++++++++---- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/tests/bdd/features/event-ledger.feature b/tests/bdd/features/event-ledger.feature index 894bb07a95..e662f579b5 100644 --- a/tests/bdd/features/event-ledger.feature +++ b/tests/bdd/features/event-ledger.feature @@ -255,21 +255,31 @@ Feature: Event ledger records SIS deployment events on a running self-managed st | --timeout | 900 | And I export the function selected by NVCF CLI to environment variables "BDD_EL_FUNCTION_ID" and "BDD_EL_VERSION_ID" - # SIS writes InstanceReady once the instance is running. + # SIS writes InstanceReady once the instance is running. Stats list every + # context in the namespace, so poll stats first. Events are stored per context, + # so the events query needs the context ids that stats reports. Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - When I poll "GET" "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/events" for up to 120 seconds until the HTTP response body contains "InstanceReady" + When I poll "GET" "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats" for up to 300 seconds until the HTTP response body contains "InstanceReady" Then the HTTP status should be 200 + And the HTTP response JSON field "namespace" should equal "${BDD_EL_VERSION_ID}" + And the HTTP response JSON field "summary.by_event.InstanceReady" should be at least 1 + When I export the HTTP response JSON field "contexts.0.context" to environment variable "BDD_EL_CONTEXT" + And I successfully run command: + """ + /bin/bash -c 'echo "${BDD_EL_CONTEXT}" | tr "," "&"' + """ + And I export command output to environment variable "BDD_EL_CONTEXT_QUERY" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/events?${BDD_EL_CONTEXT_QUERY}" + Then the HTTP status should be 200 + And the HTTP response body should contain "InstanceReady" And the HTTP response body should contain "nvidia-spot" + And the HTTP response body should contain "${BDD_EL_FUNCTION_ID}" And the HTTP response body should not contain "InstanceDestroyed" # The NVCA collector is disabled, so nothing else writes to this namespace. And the HTTP response body should not contain "ICMSRequest." - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats" - Then the HTTP status should be 200 - And the HTTP response JSON field "namespace" should equal "${BDD_EL_VERSION_ID}" - And the HTTP response JSON field "summary.by_event.InstanceReady" should be at least 1 - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats?eventFilter=InstanceReady" Then the HTTP status should be 200 @@ -280,15 +290,19 @@ Feature: Event ledger records SIS deployment events on a running self-managed st Then the HTTP status should be 200 And the HTTP response body should not contain "InstanceReady" - # Undeploy removes the instance; SIS then reports it destroyed. + # Undeploy removes the instance; SIS then reports it destroyed. Stats keep only + # the latest event per context, while events keep the whole history. When I successfully undeploy the function selected by NVCF CLI Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - When I poll "GET" "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/events" for up to 180 seconds until the HTTP response body contains "InstanceDestroyed" + When I poll "GET" "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats" for up to 180 seconds until the HTTP response body contains "InstanceDestroyed" Then the HTTP status should be 200 + And the HTTP response JSON field "summary.by_event.InstanceDestroyed" should be at least 1 Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats" - Then the HTTP response JSON field "summary.by_event.InstanceDestroyed" should be at least 1 + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/events?${BDD_EL_CONTEXT_QUERY}" + Then the HTTP status should be 200 + And the HTTP response body should contain "InstanceReady" + And the HTTP response body should contain "InstanceDestroyed" Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " And the HTTP request header "Key-Issuer-Service" is "nvcf-api" diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index bf8e926578..f56715943a 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -2949,22 +2949,28 @@ func (f *eventLedgerHTTPFake) Do(_ context.Context, method, rawURL string, heade return harness.HTTPResponse{Status: 403}, nil } namespace := strings.Split(path, "/")[4] + const eventContext = "deployment_id=d1,gpu_specification_id=g1,instance_id=i1" switch { case strings.Contains(query, "attribute_key"), strings.Contains(query, "instance_id=bad"), strings.Contains(query, "view=bogus"): return harness.HTTPResponse{Status: 400}, nil case strings.HasSuffix(path, "/stats"): - if strings.Contains(query, "NoSuchEvent") { + if namespace != "version-1" || strings.Contains(query, "NoSuchEvent") { return harness.HTTPResponse{Status: 200, Body: `{"namespace":"` + namespace + `","contexts":[]}`}, nil } - return harness.HTTPResponse{Status: 200, Body: `{"namespace":"` + namespace + `","summary":{"by_event":{"InstanceReady":1,"InstanceDestroyed":1}}}`}, nil - case namespace == "version-1": - body := `{"namespace":"version-1","events":[{"event_name":"InstanceReady","source":"nvidia-spot"}` + latest := "InstanceReady" + if *f.destroyed { + latest = "InstanceDestroyed" + } + return harness.HTTPResponse{Status: 200, Body: `{"namespace":"version-1","summary":{"by_event":{"` + latest + `":1}},"contexts":[{"context":"` + eventContext + `","event_name":"` + latest + `"}]}`}, nil + case namespace == "version-1" && strings.Contains(query, "instance_id=i1"): + body := `{"namespace":"version-1","context":"` + eventContext + `","events":[{"event_name":"InstanceReady","source":"nvidia-spot","details":{"attributes":{"functionid":"function-1"}}}` if *f.destroyed { body += `,{"event_name":"InstanceDestroyed","source":"nvidia-spot"}` } return harness.HTTPResponse{Status: 200, Body: body + `]}`}, nil default: - return harness.HTTPResponse{Status: 200, Body: `{"namespace":"` + namespace + `","events":[]}`}, nil + // Events read without a context only see context-less events. + return harness.HTTPResponse{Status: 200, Body: `{"namespace":"` + namespace + `","context":"","events":[]}`}, nil } } @@ -2987,6 +2993,10 @@ func TestEventLedgerFeatureFileWiresToSteps(t *testing.T) { Stdout: `{"currentFunction":{"hasFunction":true,"functionId":"function-1","versionId":"version-1"}}`, }, `/bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z'`: {ExitCode: 0, Stdout: "2030-01-01T00:00:00.000Z\n"}, + `/bin/bash -c 'echo "deployment_id=d1,gpu_specification_id=g1,instance_id=i1" | tr "," "&"'`: { + ExitCode: 0, + Stdout: "deployment_id=d1&gpu_specification_id=g1&instance_id=i1\n", + }, "kubectl --context k3d-ncp-local --namespace nvcf get service event-ledger -o jsonpath={.spec.ports[*].name}": { ExitCode: 0, Stdout: "api-port metrics", From 7269337c0c67470a1befa37906b7aecb49e738b4 Mon Sep 17 00:00:00 2001 From: Shelley Shen Date: Sat, 10 Oct 2026 22:30:55 -0700 Subject: [PATCH 03/11] test(bdd): drop the known-defect tag from the event-ledger scope scenario API key route scope enforcement shipped in event-ledger 0.17.1, so a read-scoped key is now rejected on the write route. Remove the known-defect tag and its stale comment, and let the read-only live entry point run the scenario: a rejected write stores no event. Signed-off-by: Shelley Shen --- tests/bdd/features/event-ledger.feature | 7 ++----- tests/bdd/godog_test.go | 5 ++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/bdd/features/event-ledger.feature b/tests/bdd/features/event-ledger.feature index e662f579b5..e2e8ebaff1 100644 --- a/tests/bdd/features/event-ledger.feature +++ b/tests/bdd/features/event-ledger.feature @@ -148,11 +148,8 @@ Feature: Event ledger records SIS deployment events on a running self-managed st When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" Then the HTTP status should be 204 - # Design: write access needs an OpenBao JWT with the FnDS write scope. A - # read-scoped api key must not be able to write. Tagged because this - # currently returns 200 and stores the event; the scenario writes data, so - # the read-only live entry point excludes it. - @known-defect + # Write access needs an OpenBao JWT with the FnDS write scope, so a + # read-scoped api key must be rejected on the write route. Scenario: A read-scoped key cannot write events When I successfully run command: """ diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index f56715943a..80f364e7fd 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -2674,14 +2674,13 @@ func TestEventLedger(t *testing.T) { } // TestEventLedgerReadOnly runs the event-ledger feature without the function -// lifecycle scenario (no GPU capacity or image pull access needed) and without -// the known-defect scenario (it writes an event). +// lifecycle scenario, so it needs no GPU capacity or image pull access. // Skipped under -short. func TestEventLedgerReadOnly(t *testing.T) { if testing.Short() { t.Skip("live run skipped under -short") } - runLiveFeatureTags(t, "event-ledger.feature", "~@function-lifecycle && ~@known-defect") + runLiveFeatureTags(t, "event-ledger.feature", "~@function-lifecycle") } // TestMultiClusterHelmfile is the live entry point for the From 13ac6a4fdbc06946d567340b840851535f61549a Mon Sep 17 00:00:00 2001 From: Shelley Shen Date: Sat, 10 Oct 2026 22:35:10 -0700 Subject: [PATCH 04/11] feat(charts): bump event-ledger release to chart v0.6.0 Pick up the 0.17.2 appVersion/image published in chart v0.6.0, which includes route scope enforcement for API key (#2302) and JWT (#2329) requests. Signed-off-by: Shelley Shen --- deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl index a5efe9bee1..4bc9273942 100644 --- a/deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl @@ -285,7 +285,7 @@ releases: - name: event-ledger chart: nvcf/helm-nvcf-event-ledger - version: 0.5.0 + version: 0.6.0 namespace: nvcf condition: addons.eventLedger.enabled values: From 29be098ca3cbf270bdb0cb700800fdece416d9c0 Mon Sep 17 00:00:00 2001 From: Shelley Shen Date: Sat, 10 Oct 2026 23:40:20 -0700 Subject: [PATCH 05/11] test(bdd): check a write-scoped event-ledger key can write but not read Add the positive control for the route scope checks: a key with only fnds:createEvent is accepted on the CloudEvents write route and refused on the stats and events read routes. This shows the existing 401 and 403 cases are specific rejections and not an outage. The event is fixed, so reruns update one row. The wiring test's fake server now tells write keys from read keys. Signed-off-by: Shelley Shen --- tests/bdd/features/event-ledger.feature | 62 +++++++++++++++++++++++++ tests/bdd/godog_test.go | 22 ++++++--- 2 files changed, 77 insertions(+), 7 deletions(-) diff --git a/tests/bdd/features/event-ledger.feature b/tests/bdd/features/event-ledger.feature index e2e8ebaff1..aab1ebf064 100644 --- a/tests/bdd/features/event-ledger.feature +++ b/tests/bdd/features/event-ledger.feature @@ -199,6 +199,68 @@ Feature: Event ledger records SIS deployment events on a running self-managed st When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" Then the HTTP status should be 204 + # The positive control for the scope checks: a key with only the write scope + # may write, which shows the rejections above are specific and not an outage, + # and it must not be able to read. The event is fixed, so reruns update one row. + Scenario: A write-scoped key can write events but cannot read them + When I successfully run command: + """ + /bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z' + """ + And I export command output to environment variable "BDD_EL_KEY_EXPIRES" + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "POST" request to "http://api-keys.localhost:8080/v1/keys" with body: + """ + { + "description": "bdd-event-ledger-write-only", + "expires_at": "${BDD_EL_KEY_EXPIRES}", + "authorizations": { + "policies": [ + { + "aud": "nvidia-event-ledger-ncp-service-id-ckozoh6f", + "auds": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"], + "product": "nv-cloud-functions", + "resources": [{"id": "*", "type": "account-functions"}], + "scopes": ["fnds:createEvent"] + } + ] + }, + "audience_service_ids": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"] + } + """ + Then the HTTP status should be 200 + When I export the HTTP response JSON field "value" to environment variable "BDD_EL_WRITE_KEY" + And I export the HTTP response JSON field "id" to environment variable "BDD_EL_WRITE_KEY_ID" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" + And the HTTP request header "Content-Type" is "application/cloudevents+json" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/cloudevents" with body: + """ + {"specversion":"1.0","id":"bdd-write-scope-1","type":"BddWriteScope","source":"bdd","namespace":"bdd-write-scope"} + """ + Then the HTTP status should be 200 + And the HTTP response JSON field "status" should equal "accepted" + And the HTTP response JSON field "success_count" should be at least 1 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-write-scope/events" + Then the HTTP status should be 403 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-write-scope/stats" + Then the HTTP status should be 403 + + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_WRITE_KEY_ID}" + Then the HTTP status should be 204 + @function-lifecycle Scenario: A function deployment produces SIS events that can be read back When I successfully run command: diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 80f364e7fd..1b7b270a43 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -2916,7 +2916,7 @@ type eventLedgerHTTPFake struct { requests []string } -func (f *eventLedgerHTTPFake) Do(_ context.Context, method, rawURL string, headers map[string]string, _ string) (harness.HTTPResponse, error) { +func (f *eventLedgerHTTPFake) Do(_ context.Context, method, rawURL string, headers map[string]string, body string) (harness.HTTPResponse, error) { f.requests = append(f.requests, method+" "+rawURL) hostAndPath := strings.TrimPrefix(rawURL, "http://") host, rest, _ := strings.Cut(hostAndPath, "/") @@ -2924,11 +2924,14 @@ func (f *eventLedgerHTTPFake) Do(_ context.Context, method, rawURL string, heade auth := headers["Authorization"] admin := auth == "Bearer admin-jwt" reader := auth == "Bearer nvapi-test" + writer := auth == "Bearer nvapi-write" if strings.HasPrefix(host, "api-keys.") { switch { case !admin || headers["Key-Issuer-Id"] == "": return harness.HTTPResponse{Status: 401}, nil + case method == "POST" && strings.Contains(body, "fnds:createEvent"): + return harness.HTTPResponse{Status: 200, Body: `{"id":"key-w","value":"nvapi-write"}`}, nil case method == "POST": return harness.HTTPResponse{Status: 200, Body: `{"id":"key-1","value":"nvapi-test"}`}, nil default: @@ -2942,6 +2945,8 @@ func (f *eventLedgerHTTPFake) Do(_ context.Context, method, rawURL string, heade return harness.HTTPResponse{Status: 404}, nil case auth == "", strings.HasPrefix(auth, "Bearer eyJ"): return harness.HTTPResponse{Status: 401}, nil + case writer && method == "POST": + return harness.HTTPResponse{Status: 200, Body: `{"status":"accepted","success_count":1,"failure_count":0,"total_processed":1}`}, nil case !reader: return harness.HTTPResponse{Status: 403}, nil case method == "POST": @@ -3040,18 +3045,21 @@ func TestEventLedgerFeatureFileWiresToSteps(t *testing.T) { if !commandRanThatContains(runs, " function delete --deployment-only") { t.Fatal("function undeploy command was never invoked") } - for _, secret := range []string{"admin-jwt", "nvapi-test"} { + for _, secret := range []string{"admin-jwt", "nvapi-test", "nvapi-write"} { if commandRanThatContains(runs, secret) { t.Fatalf("secret %q leaked into a command line", secret) } } - deletes := 0 + readKeyDeletes, writeKeyDeletes := 0, 0 for _, request := range httpFake.requests { - if request == "DELETE http://api-keys.localhost:8080/v1/keys/key-1" { - deletes++ + switch request { + case "DELETE http://api-keys.localhost:8080/v1/keys/key-1": + readKeyDeletes++ + case "DELETE http://api-keys.localhost:8080/v1/keys/key-w": + writeKeyDeletes++ } } - if deletes != 3 { - t.Fatalf("read keys deleted = %d, want 3", deletes) + if readKeyDeletes != 3 || writeKeyDeletes != 1 { + t.Fatalf("keys deleted: read = %d (want 3), write = %d (want 1)", readKeyDeletes, writeKeyDeletes) } } From 9bdc50e445f2405eda397ca3e98daddbfac06b5e Mon Sep 17 00:00:00 2001 From: Shelley Shen Date: Sat, 10 Oct 2026 23:44:14 -0700 Subject: [PATCH 06/11] test(bdd): check both event-ledger write routes enforce key scopes The cloudevents and k8s-events write routes are registered separately, so check each one: a read-scoped key is refused on both, and a write-scoped key passes the scope check on k8s-events (it then fails on the empty body with a 400, which writes nothing). Signed-off-by: Shelley Shen --- tests/bdd/features/event-ledger.feature | 15 +++++++++++++++ tests/bdd/godog_test.go | 2 ++ 2 files changed, 17 insertions(+) diff --git a/tests/bdd/features/event-ledger.feature b/tests/bdd/features/event-ledger.feature index aab1ebf064..e430942c45 100644 --- a/tests/bdd/features/event-ledger.feature +++ b/tests/bdd/features/event-ledger.feature @@ -192,6 +192,13 @@ Feature: Event ledger records SIS deployment events on a running self-managed st """ Then the HTTP status should be 401 or 403 + # The other write route is registered separately, so check it too. The scope + # check runs before the body is read, so no payload is needed. + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + And the HTTP request header "Content-Type" is "application/x-protobuf" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/k8s-events" + Then the HTTP status should be 403 + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " And the HTTP request header "Key-Issuer-Service" is "nvcf-api" And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" @@ -246,6 +253,14 @@ Feature: Event ledger records SIS deployment events on a running self-managed st And the HTTP response JSON field "status" should equal "accepted" And the HTTP response JSON field "success_count" should be at least 1 + # An empty OTLP body is rejected by the handler, so a 400 here shows the key + # passed the scope check on this route without writing anything. + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" + And the HTTP request header "Content-Type" is "application/x-protobuf" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/k8s-events" + Then the HTTP status should be 400 + And the HTTP response body should contain "No events found in request" + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-write-scope/events" Then the HTTP status should be 403 diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 1b7b270a43..4785b3cf21 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -2945,6 +2945,8 @@ func (f *eventLedgerHTTPFake) Do(_ context.Context, method, rawURL string, heade return harness.HTTPResponse{Status: 404}, nil case auth == "", strings.HasPrefix(auth, "Bearer eyJ"): return harness.HTTPResponse{Status: 401}, nil + case writer && method == "POST" && path == "/v3/ledger/k8s-events": + return harness.HTTPResponse{Status: 400, Body: `{"title":"Bad Request","status":400,"detail":"No events found in request"}`}, nil case writer && method == "POST": return harness.HTTPResponse{Status: 200, Body: `{"status":"accepted","success_count":1,"failure_count":0,"total_processed":1}`}, nil case !reader: From abe063861d473a8ca88f4094d5b425f6fb460ef5 Mon Sep 17 00:00:00 2001 From: Shelley Shen Date: Sun, 11 Oct 2026 01:17:38 -0700 Subject: [PATCH 07/11] test(bdd): find the SIS context with an event filter in the lifecycle scenario The NVCA OTel collector is off by default. When it is enabled, NVCA adds contexts that sort before the SIS one in stats, so reading the first context misses the SIS events. Select the SIS context with an event filter instead, so the scenario works with the collector on or off. The wiring test's fake keeps one NVCA context to cover that. Signed-off-by: Shelley Shen --- tests/bdd/features/event-ledger.feature | 28 ++++---- tests/bdd/godog_test.go | 92 ++++++++++++++++++++----- 2 files changed, 89 insertions(+), 31 deletions(-) diff --git a/tests/bdd/features/event-ledger.feature b/tests/bdd/features/event-ledger.feature index e430942c45..cd80e7f818 100644 --- a/tests/bdd/features/event-ledger.feature +++ b/tests/bdd/features/event-ledger.feature @@ -6,8 +6,8 @@ Feature: Event ledger records SIS deployment events on a running self-managed st # This feature does not install anything. It expects the single-cluster # ncp-local stack to be running with addons.eventLedger.enabled=true and - # nvcf-cli initialized for tests/bdd/fixtures/nvcf-cli-local.yaml. The - # NVCA OTel collector is disabled, so SIS is the only event writer. + # nvcf-cli initialized for tests/bdd/fixtures/nvcf-cli-local.yaml. It checks + # the SIS events only and does not depend on the NVCA OTel collector. Background: Given these environment variables are set: | name | @@ -329,14 +329,21 @@ Feature: Event ledger records SIS deployment events on a running self-managed st | --timeout | 900 | And I export the function selected by NVCF CLI to environment variables "BDD_EL_FUNCTION_ID" and "BDD_EL_VERSION_ID" - # SIS writes InstanceReady once the instance is running. Stats list every - # context in the namespace, so poll stats first. Events are stored per context, - # so the events query needs the context ids that stats reports. + # Events are stored per context, and stats list every context in the + # namespace with its latest event. The NVCA collector, when enabled, adds + # contexts of its own that sort before SIS's, so find the SIS context with + # an event filter, then read its events. + + # SIS writes InstanceReady once the instance is running. Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" When I poll "GET" "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats" for up to 300 seconds until the HTTP response body contains "InstanceReady" Then the HTTP status should be 200 And the HTTP response JSON field "namespace" should equal "${BDD_EL_VERSION_ID}" And the HTTP response JSON field "summary.by_event.InstanceReady" should be at least 1 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats?eventFilter=InstanceReady" + Then the HTTP status should be 200 When I export the HTTP response JSON field "contexts.0.context" to environment variable "BDD_EL_CONTEXT" And I successfully run command: """ @@ -351,21 +358,14 @@ Feature: Event ledger records SIS deployment events on a running self-managed st And the HTTP response body should contain "nvidia-spot" And the HTTP response body should contain "${BDD_EL_FUNCTION_ID}" And the HTTP response body should not contain "InstanceDestroyed" - # The NVCA collector is disabled, so nothing else writes to this namespace. - And the HTTP response body should not contain "ICMSRequest." - - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats?eventFilter=InstanceReady" - Then the HTTP status should be 200 - And the HTTP response body should contain "InstanceReady" Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats?eventFilter=NoSuchEvent" Then the HTTP status should be 200 And the HTTP response body should not contain "InstanceReady" - # Undeploy removes the instance; SIS then reports it destroyed. Stats keep only - # the latest event per context, while events keep the whole history. + # Undeploy removes the instance and SIS then reports it destroyed. Stats keep + # only the latest event per context, while events keep the whole history. When I successfully undeploy the function selected by NVCF CLI Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" When I poll "GET" "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats" for up to 180 seconds until the HTTP response body contains "InstanceDestroyed" diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 4785b3cf21..4bdc52d62a 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -21,7 +21,9 @@ package bdd_tmp import ( "context" + "encoding/json" "io" + "net/url" "os" "path/filepath" "strings" @@ -2955,29 +2957,85 @@ func (f *eventLedgerHTTPFake) Do(_ context.Context, method, rawURL string, heade return harness.HTTPResponse{Status: 403}, nil } namespace := strings.Split(path, "/")[4] - const eventContext = "deployment_id=d1,gpu_specification_id=g1,instance_id=i1" + params, _ := url.ParseQuery(query) switch { - case strings.Contains(query, "attribute_key"), strings.Contains(query, "instance_id=bad"), strings.Contains(query, "view=bogus"): + case params.Has("attribute_key") || strings.HasPrefix(params.Get("instance_id"), "bad") || params.Has("view"): return harness.HTTPResponse{Status: 400}, nil case strings.HasSuffix(path, "/stats"): - if namespace != "version-1" || strings.Contains(query, "NoSuchEvent") { - return harness.HTTPResponse{Status: 200, Body: `{"namespace":"` + namespace + `","contexts":[]}`}, nil + return f.stats(namespace, params.Get("eventFilter")), nil + default: + return f.events(namespace, params), nil + } +} + +// Contexts the fake keeps for the function version "version-1": the SIS one +// and an NVCA one that sorts before it, as the ledger lists them when the NVCA +// collector is enabled. The scenario must still pick the SIS context. +const ( + fakeNVCAContext = "cluster_id=c1,instance_id=i1,resource_id=r1" + fakeSISContext = "deployment_id=d1,gpu_specification_id=g1,instance_id=i1" +) + +type fakeEvent struct{ name, source string } + +func (f *eventLedgerHTTPFake) contextEvents() map[string][]fakeEvent { + sis := []fakeEvent{{"InstanceReady", "nvidia-spot"}} + if *f.destroyed { + sis = append(sis, fakeEvent{"InstanceDestroyed", "nvidia-spot"}) + } + return map[string][]fakeEvent{ + fakeNVCAContext: {{"ICMSRequest.InstanceStatusUpdate.running", "nvca"}}, + fakeSISContext: sis, + } +} + +func (f *eventLedgerHTTPFake) stats(namespace, eventFilter string) harness.HTTPResponse { + type row struct { + Context string `json:"context"` + EventName string `json:"event_name"` + } + contexts := []row{} + byEvent := map[string]int{} + if namespace == "version-1" { + events := f.contextEvents() + for _, ctx := range []string{fakeNVCAContext, fakeSISContext} { + latest := events[ctx][len(events[ctx])-1].name + if eventFilter != "" && !strings.Contains(","+eventFilter+",", ","+latest+",") { + continue + } + contexts = append(contexts, row{ctx, latest}) + byEvent[latest]++ } - latest := "InstanceReady" - if *f.destroyed { - latest = "InstanceDestroyed" + } + body, _ := json.Marshal(map[string]any{ + "namespace": namespace, + "summary": map[string]any{"by_event": byEvent}, + "contexts": contexts, + }) + return harness.HTTPResponse{Status: 200, Body: string(body)} +} + +func (f *eventLedgerHTTPFake) events(namespace string, params url.Values) harness.HTTPResponse { + var parts []string + for _, key := range []string{"cluster_id", "deployment_id", "gpu_specification_id", "instance_id", "resource_id"} { + if value := params.Get(key); value != "" { + parts = append(parts, key+"="+value) } - return harness.HTTPResponse{Status: 200, Body: `{"namespace":"version-1","summary":{"by_event":{"` + latest + `":1}},"contexts":[{"context":"` + eventContext + `","event_name":"` + latest + `"}]}`}, nil - case namespace == "version-1" && strings.Contains(query, "instance_id=i1"): - body := `{"namespace":"version-1","context":"` + eventContext + `","events":[{"event_name":"InstanceReady","source":"nvidia-spot","details":{"attributes":{"functionid":"function-1"}}}` - if *f.destroyed { - body += `,{"event_name":"InstanceDestroyed","source":"nvidia-spot"}` + } + eventContext := strings.Join(parts, ",") + type event struct { + Name string `json:"event_name"` + Source string `json:"source"` + Details map[string]any `json:"details"` + } + events := []event{} + if namespace == "version-1" { + for _, e := range f.contextEvents()[eventContext] { + events = append(events, event{e.name, e.source, map[string]any{"attributes": map[string]string{"functionid": "function-1", "function_id": "function-1"}}}) } - return harness.HTTPResponse{Status: 200, Body: body + `]}`}, nil - default: - // Events read without a context only see context-less events. - return harness.HTTPResponse{Status: 200, Body: `{"namespace":"` + namespace + `","context":"","events":[]}`}, nil } + body, _ := json.Marshal(map[string]any{"namespace": namespace, "context": eventContext, "events": events}) + return harness.HTTPResponse{Status: 200, Body: string(body)} } func TestEventLedgerFeatureFileWiresToSteps(t *testing.T) { @@ -2999,7 +3057,7 @@ func TestEventLedgerFeatureFileWiresToSteps(t *testing.T) { Stdout: `{"currentFunction":{"hasFunction":true,"functionId":"function-1","versionId":"version-1"}}`, }, `/bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z'`: {ExitCode: 0, Stdout: "2030-01-01T00:00:00.000Z\n"}, - `/bin/bash -c 'echo "deployment_id=d1,gpu_specification_id=g1,instance_id=i1" | tr "," "&"'`: { + `/bin/bash -c 'echo "` + fakeSISContext + `" | tr "," "&"'`: { ExitCode: 0, Stdout: "deployment_id=d1&gpu_specification_id=g1&instance_id=i1\n", }, From 1b4cef26e1e2642fa3c18473cd6fb2f70b4c1cc5 Mon Sep 17 00:00:00 2001 From: Shelley Shen Date: Sun, 11 Oct 2026 13:02:41 -0700 Subject: [PATCH 08/11] test(bdd): give API Keys a longer startup window in the local fixture On a slow host the API Keys app takes longer than its default 3 minute startup window to start. The kubelet restarts it before it is ready, so the control-plane install waits on it until Helm times out. Set apikeys.startupProbe.failureThreshold to 120 as a native integer. The Helmfile environment table step writes every value as a string, and the stack passes this value through to the Deployment as is, so it has to live in the fixture. Signed-off-by: Shelley Shen --- tests/bdd/fixtures/self-managed-local-bdd.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/bdd/fixtures/self-managed-local-bdd.yaml b/tests/bdd/fixtures/self-managed-local-bdd.yaml index 9e733e4f1b..c75eac8c9c 100644 --- a/tests/bdd/fixtures/self-managed-local-bdd.yaml +++ b/tests/bdd/fixtures/self-managed-local-bdd.yaml @@ -101,3 +101,8 @@ ingress: enabled: true ess: enabled: true + +# API Keys can take longer than its default 3 minute startup window on a slow host. +apikeys: + startupProbe: + failureThreshold: 120 From b7cc5ee9760c38f93c7419407520cbbf7a3b9afd Mon Sep 17 00:00:00 2001 From: Shelley Shen Date: Sun, 11 Oct 2026 13:02:41 -0700 Subject: [PATCH 09/11] test(bdd): install the stack in the event-ledger feature Install the stack the way the other features do instead of attaching to a running one: prepare the Helmfile environment, bring up the single cluster, install the control plane, run nvcf-cli init, then check the event ledger. Register the cluster and install NVCA afterwards, which the function deployment scenario needs. TestEventLedger runs the whole feature. TestEventLedgerControlPlane skips the compute-plane scenarios, so it needs no GPU capacity or image pull access. The wiring test now covers the install steps. Signed-off-by: Shelley Shen --- tests/bdd/features/event-ledger.feature | 884 +++++++++++++----------- tests/bdd/godog_test.go | 50 +- 2 files changed, 543 insertions(+), 391 deletions(-) diff --git a/tests/bdd/features/event-ledger.feature b/tests/bdd/features/event-ledger.feature index cd80e7f818..4eda2c434c 100644 --- a/tests/bdd/features/event-ledger.feature +++ b/tests/bdd/features/event-ledger.feature @@ -1,386 +1,500 @@ -@event-ledger @ncp-local @single-cluster -Feature: Event ledger records SIS deployment events on a running self-managed stack +@event-ledger @ncp-local @single-cluster @helmfile +Feature: Install a local single-cluster NVCF stack and check the event ledger As a self-managed NVCF operator, - I want the event ledger to be reachable, protected, and fed by SIS, - so that I can read the lifecycle events of my function deployments. - - # This feature does not install anything. It expects the single-cluster - # ncp-local stack to be running with addons.eventLedger.enabled=true and - # nvcf-cli initialized for tests/bdd/fixtures/nvcf-cli-local.yaml. It checks - # the SIS events only and does not depend on the NVCA OTel collector. - Background: - Given these environment variables are set: - | name | - | NGC_API_KEY | - | SAMPLE_NGC_ORG | - | SAMPLE_NGC_TEAM | - | REPO_ROOT | - And I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml" - - Scenario: Event ledger is running and routed through the shared gateway - Then deployment "event-ledger" in namespace "nvcf" using context "k3d-ncp-local" should complete rollout within "2m" - And these Kubernetes resources should exist in namespace "nvcf" using context "k3d-ncp-local": - | kind | name | - | service | event-ledger | - And these Gateway API routes should be accepted and resolved using context "k3d-ncp-local" within "2m": - | kind | name | namespace | parent | - | HTTPRoute | event-ledger | envoy-gateway-system | shared-gw | - - When I successfully run command "kubectl --context k3d-ncp-local --namespace nvcf get service event-ledger -o jsonpath={.spec.ports[*].name}" - Then the command output should contain "api-port" - - When I send a "GET" request to "http://events.localhost:8080/health" - Then the HTTP status should be 200 - - Scenario: SIS is configured to send events to the event ledger - When I successfully run command "kubectl --context k3d-ncp-local --namespace sis get configmaps -o yaml" - Then the command output should contain all: - | text | - | ICMS_FNDS_MESSAGES_ENABLED: "true" | - | ICMS_FNDS_MESSAGES_V3_ENABLED: "true" | - - # A missing or JWT-shaped invalid credential is rejected with 401. An opaque - # token is not a JWT, so it goes to the api-keys policy evaluator, which - # denies an unknown key with 403. - Scenario: Requests without valid credentials are rejected - When I send a "POST" request to "http://events.localhost:8080/v3/ledger/cloudevents" with body: - """ - {"specversion":"1.0"} - """ - Then the HTTP status should be 401 - - Given the HTTP request header "Authorization" is "Bearer eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ4In0.c2ln" - When I send a "POST" request to "http://events.localhost:8080/v3/ledger/cloudevents" - Then the HTTP status should be 401 - - Given the HTTP request header "Authorization" is "Bearer not-a-token" - When I send a "POST" request to "http://events.localhost:8080/v3/ledger/k8s-events" - Then the HTTP status should be 403 - - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-auth/stats" - Then the HTTP status should be 401 - - Given the HTTP request header "Authorization" is "Bearer eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ4In0.c2ln" - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-auth/events" - Then the HTTP status should be 401 - - Given the HTTP request header "Authorization" is "Bearer nvapi-not-a-real-key" - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-auth/events" - Then the HTTP status should be 403 - - When I send a "GET" request to "http://events.localhost:8080/status" - Then the HTTP status should be 401 - - Scenario: V1 and V2 endpoints are disabled - When I send a "POST" request to "http://events.localhost:8080/v1/ledger/versions/bdd-version/instances/bdd-instance" with body: - """ - {} - """ - Then the HTTP status should be 404 - - When I send a "POST" request to "http://events.localhost:8080/v2/ledger/versions/bdd-version/deployments/bdd-deployment/instances/bdd-instance" with body: - """ - {} - """ - Then the HTTP status should be 404 - - Scenario: A read key can query the ledger and bad queries are rejected - When I successfully run command: - """ - /bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z' - """ - And I export command output to environment variable "BDD_EL_KEY_EXPIRES" - Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " - And the HTTP request header "Content-Type" is "application/json" - And the HTTP request header "Key-Issuer-Service" is "nvcf-api" - And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" - And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" - When I send a "POST" request to "http://api-keys.localhost:8080/v1/keys" with body: - """ - { - "description": "bdd-event-ledger-read", - "expires_at": "${BDD_EL_KEY_EXPIRES}", - "authorizations": { - "policies": [ - { - "aud": "nvidia-event-ledger-ncp-service-id-ckozoh6f", - "auds": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"], - "product": "nv-cloud-functions", - "resources": [{"id": "*", "type": "account-functions"}], - "scopes": ["fnds:getEvents", "fnds:getStats"] - } - ] - }, - "audience_service_ids": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"] - } - """ - Then the HTTP status should be 200 - When I export the HTTP response JSON field "value" to environment variable "BDD_EL_READ_KEY" - And I export the HTTP response JSON field "id" to environment variable "BDD_EL_READ_KEY_ID" - - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/stats" - Then the HTTP status should be 200 - And the HTTP response JSON field "namespace" should equal "bdd-no-such-namespace" - - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/events" - Then the HTTP status should be 200 - - # Malformed queries must get a 400 before reaching Cassandra: a half-supplied - # attribute filter, a disallowed character in a context id, and an unknown stats view. - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/events?attribute_key=only-a-key" - Then the HTTP status should be 400 - - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/events?instance_id=bad!id" - Then the HTTP status should be 400 - - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/stats?view=bogus" - Then the HTTP status should be 400 - - Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " - And the HTTP request header "Key-Issuer-Service" is "nvcf-api" - And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" - And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" - When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" - Then the HTTP status should be 204 - - # Write access needs an OpenBao JWT with the FnDS write scope, so a - # read-scoped api key must be rejected on the write route. - Scenario: A read-scoped key cannot write events - When I successfully run command: - """ - /bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z' - """ - And I export command output to environment variable "BDD_EL_KEY_EXPIRES" - Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " - And the HTTP request header "Content-Type" is "application/json" - And the HTTP request header "Key-Issuer-Service" is "nvcf-api" - And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" - And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" - When I send a "POST" request to "http://api-keys.localhost:8080/v1/keys" with body: - """ - { - "description": "bdd-event-ledger-read-only-write", - "expires_at": "${BDD_EL_KEY_EXPIRES}", - "authorizations": { - "policies": [ - { - "aud": "nvidia-event-ledger-ncp-service-id-ckozoh6f", - "auds": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"], - "product": "nv-cloud-functions", - "resources": [{"id": "*", "type": "account-functions"}], - "scopes": ["fnds:getEvents", "fnds:getStats"] - } - ] - }, - "audience_service_ids": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"] - } - """ - Then the HTTP status should be 200 - When I export the HTTP response JSON field "value" to environment variable "BDD_EL_READ_KEY" - And I export the HTTP response JSON field "id" to environment variable "BDD_EL_READ_KEY_ID" - - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - And the HTTP request header "Content-Type" is "application/cloudevents+json" - When I send a "POST" request to "http://events.localhost:8080/v3/ledger/cloudevents" with body: - """ - {"specversion":"1.0","id":"bdd-write-attempt-1","type":"BddWriteAttempt","source":"bdd","namespace":"bdd-write-attempt"} - """ - Then the HTTP status should be 401 or 403 - - # The other write route is registered separately, so check it too. The scope - # check runs before the body is read, so no payload is needed. - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - And the HTTP request header "Content-Type" is "application/x-protobuf" - When I send a "POST" request to "http://events.localhost:8080/v3/ledger/k8s-events" - Then the HTTP status should be 403 - - Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " - And the HTTP request header "Key-Issuer-Service" is "nvcf-api" - And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" - And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" - When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" - Then the HTTP status should be 204 - - # The positive control for the scope checks: a key with only the write scope - # may write, which shows the rejections above are specific and not an outage, - # and it must not be able to read. The event is fixed, so reruns update one row. - Scenario: A write-scoped key can write events but cannot read them - When I successfully run command: - """ - /bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z' - """ - And I export command output to environment variable "BDD_EL_KEY_EXPIRES" - Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " - And the HTTP request header "Content-Type" is "application/json" - And the HTTP request header "Key-Issuer-Service" is "nvcf-api" - And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" - And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" - When I send a "POST" request to "http://api-keys.localhost:8080/v1/keys" with body: - """ - { - "description": "bdd-event-ledger-write-only", - "expires_at": "${BDD_EL_KEY_EXPIRES}", - "authorizations": { - "policies": [ - { - "aud": "nvidia-event-ledger-ncp-service-id-ckozoh6f", - "auds": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"], - "product": "nv-cloud-functions", - "resources": [{"id": "*", "type": "account-functions"}], - "scopes": ["fnds:createEvent"] - } - ] - }, - "audience_service_ids": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"] - } - """ - Then the HTTP status should be 200 - When I export the HTTP response JSON field "value" to environment variable "BDD_EL_WRITE_KEY" - And I export the HTTP response JSON field "id" to environment variable "BDD_EL_WRITE_KEY_ID" - - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" - And the HTTP request header "Content-Type" is "application/cloudevents+json" - When I send a "POST" request to "http://events.localhost:8080/v3/ledger/cloudevents" with body: - """ - {"specversion":"1.0","id":"bdd-write-scope-1","type":"BddWriteScope","source":"bdd","namespace":"bdd-write-scope"} - """ - Then the HTTP status should be 200 - And the HTTP response JSON field "status" should equal "accepted" - And the HTTP response JSON field "success_count" should be at least 1 - - # An empty OTLP body is rejected by the handler, so a 400 here shows the key - # passed the scope check on this route without writing anything. - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" - And the HTTP request header "Content-Type" is "application/x-protobuf" - When I send a "POST" request to "http://events.localhost:8080/v3/ledger/k8s-events" - Then the HTTP status should be 400 - And the HTTP response body should contain "No events found in request" - - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-write-scope/events" - Then the HTTP status should be 403 - - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-write-scope/stats" - Then the HTTP status should be 403 - - Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " - And the HTTP request header "Key-Issuer-Service" is "nvcf-api" - And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" - And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" - When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_WRITE_KEY_ID}" - Then the HTTP status should be 204 - - @function-lifecycle - Scenario: A function deployment produces SIS events that can be read back - When I successfully run command: - """ - /bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z' - """ - And I export command output to environment variable "BDD_EL_KEY_EXPIRES" - Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " - And the HTTP request header "Content-Type" is "application/json" - And the HTTP request header "Key-Issuer-Service" is "nvcf-api" - And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" - And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" - When I send a "POST" request to "http://api-keys.localhost:8080/v1/keys" with body: - """ - { - "description": "bdd-event-ledger-lifecycle", - "expires_at": "${BDD_EL_KEY_EXPIRES}", - "authorizations": { - "policies": [ - { - "aud": "nvidia-event-ledger-ncp-service-id-ckozoh6f", - "auds": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"], - "product": "nv-cloud-functions", - "resources": [{"id": "*", "type": "account-functions"}], - "scopes": ["fnds:getEvents", "fnds:getStats"] - } - ] - }, - "audience_service_ids": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"] - } - """ - Then the HTTP status should be 200 - When I export the HTTP response JSON field "value" to environment variable "BDD_EL_READ_KEY" - And I export the HTTP response JSON field "id" to environment variable "BDD_EL_READ_KEY_ID" - - When I successfully create function "bdd-event-ledger" from image "nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/load_tester_supreme:0.0.8" with CLI options: - | option | value | - | --inference-url | /echo | - | --inference-port | 8000 | - | --health-uri | /health | - | --health-port | 8000 | - | --health-timeout | PT30S | - And I successfully deploy the function selected by NVCF CLI with options: - | option | value | - | --gpu | H100 | - | --instance-type | NCP.GPU.H100_1x | - | --backend | ncp-local | - | --regions | us-west-1 | - | --min-instances | 1 | - | --max-instances | 1 | - | --timeout | 900 | - And I export the function selected by NVCF CLI to environment variables "BDD_EL_FUNCTION_ID" and "BDD_EL_VERSION_ID" - - # Events are stored per context, and stats list every context in the - # namespace with its latest event. The NVCA collector, when enabled, adds - # contexts of its own that sort before SIS's, so find the SIS context with - # an event filter, then read its events. - - # SIS writes InstanceReady once the instance is running. - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - When I poll "GET" "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats" for up to 300 seconds until the HTTP response body contains "InstanceReady" - Then the HTTP status should be 200 - And the HTTP response JSON field "namespace" should equal "${BDD_EL_VERSION_ID}" - And the HTTP response JSON field "summary.by_event.InstanceReady" should be at least 1 - - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats?eventFilter=InstanceReady" - Then the HTTP status should be 200 - When I export the HTTP response JSON field "contexts.0.context" to environment variable "BDD_EL_CONTEXT" - And I successfully run command: - """ - /bin/bash -c 'echo "${BDD_EL_CONTEXT}" | tr "," "&"' - """ - And I export command output to environment variable "BDD_EL_CONTEXT_QUERY" - - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/events?${BDD_EL_CONTEXT_QUERY}" - Then the HTTP status should be 200 - And the HTTP response body should contain "InstanceReady" - And the HTTP response body should contain "nvidia-spot" - And the HTTP response body should contain "${BDD_EL_FUNCTION_ID}" - And the HTTP response body should not contain "InstanceDestroyed" - - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats?eventFilter=NoSuchEvent" - Then the HTTP status should be 200 - And the HTTP response body should not contain "InstanceReady" - - # Undeploy removes the instance and SIS then reports it destroyed. Stats keep - # only the latest event per context, while events keep the whole history. - When I successfully undeploy the function selected by NVCF CLI - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - When I poll "GET" "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats" for up to 180 seconds until the HTTP response body contains "InstanceDestroyed" - Then the HTTP status should be 200 - And the HTTP response JSON field "summary.by_event.InstanceDestroyed" should be at least 1 - - Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" - When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/events?${BDD_EL_CONTEXT_QUERY}" - Then the HTTP status should be 200 - And the HTTP response body should contain "InstanceReady" - And the HTTP response body should contain "InstanceDestroyed" - - Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " - And the HTTP request header "Key-Issuer-Service" is "nvcf-api" - And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" - And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" - When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" - Then the HTTP status should be 204 + I want to install the stack with the event ledger enabled, + so that I can check it is reachable, protected, and fed by SIS. + + Rule: Helmfile installs the local control plane with the event ledger enabled + + Background: + Given these environment variables are set: + | name | + | NGC_API_KEY | + | SAMPLE_NGC_ORG | + | SAMPLE_NGC_TEAM | + | NVCF_CLI | + | REPO_ROOT | + # The fixture is a copy of deploy/stacks/self-managed/environments/local.yaml, + # which already carries every ncp-local local-mode override. The Background + # only overlays the values that vary per NGC org and pull-secret name. The + # event ledger add-on is on by default in the stack's base environment. + And I prepare Helmfile environment "local-bdd-event-ledger" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd.yaml" with values: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + And I prepare Helmfile environment "local-bdd-event-ledger" for stack "nvcf-compute-plane" from fixture "tests/bdd/fixtures/nvcf-compute-plane-local-bdd.yaml" with values: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-event-ledger-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential + # Conflict precheck: ncp-local-cp's k3d serverlb claims + # 0.0.0.0:8080/8443/10081, NATS on 4222, and the worker callback port + # 10086, overlapping host ports single-cluster ncp-local needs. Fail loudly + # so the operator runs `make -C tools/ncp-local-cluster destroy-multicluster` + # before retrying. `k3d cluster get` exits 1 when absent (k3d v5). + And I run command "k3d cluster get ncp-local-cp" + And the command exit code should be 1 + And a single-cluster ncp-local cluster is running + And the "nvcr-pull-secret" image pull secret exists in namespaces: + | cassandra-system | + | nats-system | + | nvcf | + | api-keys | + | ess | + | sis | + | vault-system | + | nvca-operator | + | cert-manager | + + Scenario: Operator installs the control plane with the event ledger enabled + When I successfully run command "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-event-ledger" + + Then these Helm releases should be deployed using context "k3d-ncp-local": + | name | namespace | + | nats | nats-system | + | cert-manager | cert-manager | + | openbao-server | vault-system | + | cassandra | cassandra-system | + | api-keys | api-keys | + | sis | sis | + | api | nvcf | + | event-ledger | nvcf | + | ingress | envoy-gateway-system | + + # The checks below create API keys, which needs the admin token. + When I successfully run command "${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml init" + + Rule: The event ledger is reachable, protected, and readable + + Background: + Given these environment variables are set: + | name | + | NGC_API_KEY | + | SAMPLE_NGC_ORG | + | SAMPLE_NGC_TEAM | + | REPO_ROOT | + And I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml" + + # These scenarios depend on the earlier control-plane install scenario in the + # same feature run and are not standalone tag targets. + Scenario: Event ledger is running and routed through the shared gateway + Then deployment "event-ledger" in namespace "nvcf" using context "k3d-ncp-local" should complete rollout within "2m" + And these Kubernetes resources should exist in namespace "nvcf" using context "k3d-ncp-local": + | kind | name | + | service | event-ledger | + And these Gateway API routes should be accepted and resolved using context "k3d-ncp-local" within "2m": + | kind | name | namespace | parent | + | HTTPRoute | event-ledger | envoy-gateway-system | shared-gw | + + When I successfully run command "kubectl --context k3d-ncp-local --namespace nvcf get service event-ledger -o jsonpath={.spec.ports[*].name}" + Then the command output should contain "api-port" + + When I send a "GET" request to "http://events.localhost:8080/health" + Then the HTTP status should be 200 + + Scenario: SIS is configured to send events to the event ledger + When I successfully run command "kubectl --context k3d-ncp-local --namespace sis get configmaps -o yaml" + Then the command output should contain all: + | text | + | ICMS_FNDS_MESSAGES_ENABLED: "true" | + | ICMS_FNDS_MESSAGES_V3_ENABLED: "true" | + + # A missing or JWT-shaped invalid credential is rejected with 401. An opaque + # token is not a JWT, so it goes to the api-keys policy evaluator, which + # denies an unknown key with 403. + Scenario: Requests without valid credentials are rejected + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/cloudevents" with body: + """ + {"specversion":"1.0"} + """ + Then the HTTP status should be 401 + + Given the HTTP request header "Authorization" is "Bearer eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ4In0.c2ln" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/cloudevents" + Then the HTTP status should be 401 + + Given the HTTP request header "Authorization" is "Bearer not-a-token" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/k8s-events" + Then the HTTP status should be 403 + + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-auth/stats" + Then the HTTP status should be 401 + + Given the HTTP request header "Authorization" is "Bearer eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ4In0.c2ln" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-auth/events" + Then the HTTP status should be 401 + + Given the HTTP request header "Authorization" is "Bearer nvapi-not-a-real-key" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-auth/events" + Then the HTTP status should be 403 + + When I send a "GET" request to "http://events.localhost:8080/status" + Then the HTTP status should be 401 + + Scenario: V1 and V2 endpoints are disabled + When I send a "POST" request to "http://events.localhost:8080/v1/ledger/versions/bdd-version/instances/bdd-instance" with body: + """ + {} + """ + Then the HTTP status should be 404 + + When I send a "POST" request to "http://events.localhost:8080/v2/ledger/versions/bdd-version/deployments/bdd-deployment/instances/bdd-instance" with body: + """ + {} + """ + Then the HTTP status should be 404 + + Scenario: A read key can query the ledger and bad queries are rejected + When I successfully run command: + """ + /bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z' + """ + And I export command output to environment variable "BDD_EL_KEY_EXPIRES" + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "POST" request to "http://api-keys.localhost:8080/v1/keys" with body: + """ + { + "description": "bdd-event-ledger-read", + "expires_at": "${BDD_EL_KEY_EXPIRES}", + "authorizations": { + "policies": [ + { + "aud": "nvidia-event-ledger-ncp-service-id-ckozoh6f", + "auds": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"], + "product": "nv-cloud-functions", + "resources": [{"id": "*", "type": "account-functions"}], + "scopes": ["fnds:getEvents", "fnds:getStats"] + } + ] + }, + "audience_service_ids": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"] + } + """ + Then the HTTP status should be 200 + When I export the HTTP response JSON field "value" to environment variable "BDD_EL_READ_KEY" + And I export the HTTP response JSON field "id" to environment variable "BDD_EL_READ_KEY_ID" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/stats" + Then the HTTP status should be 200 + And the HTTP response JSON field "namespace" should equal "bdd-no-such-namespace" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/events" + Then the HTTP status should be 200 + + # Malformed queries must get a 400 before reaching Cassandra: a half-supplied + # attribute filter, a disallowed character in a context id, and an unknown stats view. + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/events?attribute_key=only-a-key" + Then the HTTP status should be 400 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/events?instance_id=bad!id" + Then the HTTP status should be 400 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/stats?view=bogus" + Then the HTTP status should be 400 + + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" + Then the HTTP status should be 204 + + # Write access needs an OpenBao JWT with the FnDS write scope, so a + # read-scoped api key must be rejected on the write route. + Scenario: A read-scoped key cannot write events + When I successfully run command: + """ + /bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z' + """ + And I export command output to environment variable "BDD_EL_KEY_EXPIRES" + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "POST" request to "http://api-keys.localhost:8080/v1/keys" with body: + """ + { + "description": "bdd-event-ledger-read-only-write", + "expires_at": "${BDD_EL_KEY_EXPIRES}", + "authorizations": { + "policies": [ + { + "aud": "nvidia-event-ledger-ncp-service-id-ckozoh6f", + "auds": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"], + "product": "nv-cloud-functions", + "resources": [{"id": "*", "type": "account-functions"}], + "scopes": ["fnds:getEvents", "fnds:getStats"] + } + ] + }, + "audience_service_ids": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"] + } + """ + Then the HTTP status should be 200 + When I export the HTTP response JSON field "value" to environment variable "BDD_EL_READ_KEY" + And I export the HTTP response JSON field "id" to environment variable "BDD_EL_READ_KEY_ID" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + And the HTTP request header "Content-Type" is "application/cloudevents+json" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/cloudevents" with body: + """ + {"specversion":"1.0","id":"bdd-write-attempt-1","type":"BddWriteAttempt","source":"bdd","namespace":"bdd-write-attempt"} + """ + Then the HTTP status should be 401 or 403 + + # The other write route is registered separately, so check it too. The scope + # check runs before the body is read, so no payload is needed. + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + And the HTTP request header "Content-Type" is "application/x-protobuf" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/k8s-events" + Then the HTTP status should be 403 + + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" + Then the HTTP status should be 204 + + # The positive control for the scope checks: a key with only the write scope + # may write, which shows the rejections above are specific and not an outage, + # and it must not be able to read. The event is fixed, so reruns update one row. + Scenario: A write-scoped key can write events but cannot read them + When I successfully run command: + """ + /bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z' + """ + And I export command output to environment variable "BDD_EL_KEY_EXPIRES" + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "POST" request to "http://api-keys.localhost:8080/v1/keys" with body: + """ + { + "description": "bdd-event-ledger-write-only", + "expires_at": "${BDD_EL_KEY_EXPIRES}", + "authorizations": { + "policies": [ + { + "aud": "nvidia-event-ledger-ncp-service-id-ckozoh6f", + "auds": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"], + "product": "nv-cloud-functions", + "resources": [{"id": "*", "type": "account-functions"}], + "scopes": ["fnds:createEvent"] + } + ] + }, + "audience_service_ids": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"] + } + """ + Then the HTTP status should be 200 + When I export the HTTP response JSON field "value" to environment variable "BDD_EL_WRITE_KEY" + And I export the HTTP response JSON field "id" to environment variable "BDD_EL_WRITE_KEY_ID" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" + And the HTTP request header "Content-Type" is "application/cloudevents+json" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/cloudevents" with body: + """ + {"specversion":"1.0","id":"bdd-write-scope-1","type":"BddWriteScope","source":"bdd","namespace":"bdd-write-scope"} + """ + Then the HTTP status should be 200 + And the HTTP response JSON field "status" should equal "accepted" + And the HTTP response JSON field "success_count" should be at least 1 + + # An empty OTLP body is rejected by the handler, so a 400 here shows the key + # passed the scope check on this route without writing anything. + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" + And the HTTP request header "Content-Type" is "application/x-protobuf" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/k8s-events" + Then the HTTP status should be 400 + And the HTTP response body should contain "No events found in request" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-write-scope/events" + Then the HTTP status should be 403 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-write-scope/stats" + Then the HTTP status should be 403 + + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_WRITE_KEY_ID}" + Then the HTTP status should be 204 + + Rule: Helmfile installs NVCA on the same local cluster after registration + + Background: + Given these environment variables are set: + | name | + | NVCF_CLI | + | REPO_ROOT | + # This rule depends on the earlier control-plane install scenario in the + # same feature run. It is only needed for the function deployment scenario. + + @nvca-registration @compute-plane + Scenario: Operator registers the local cluster and installs the NVCA operator + When I successfully run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml self-hosted --control-plane-stack deploy/stacks/self-managed --env local-bdd-event-ledger control-plane profile export --cluster-name ncp-local + """ + Then file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should exist + + When I successfully run command: + """ + make -C deploy/stacks/nvcf-compute-plane register-cluster CLUSTER_NAME=ncp-local CONTROL_PLANE_PROFILE=${REPO_ROOT}/deploy/stacks/self-managed/out/control-plane-profile.yaml COMPUTE_KUBE_CONTEXT=k3d-ncp-local NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml + """ + Then file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-register-values.yaml" should exist + And yaml file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-register-values.yaml" should have non-empty keys: + | key | + | clusterID | + | clusterGroupID | + + When I successfully run command: + """ + make -C deploy/stacks/nvcf-compute-plane install CLUSTER_NAME=ncp-local HELMFILE_ENV=local-bdd-event-ledger NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml + """ + + Then these Helm releases should be deployed using context "k3d-ncp-local": + | name | namespace | + | nvca-operator | nvca-operator | + + Then deployment "nvca-operator" in namespace "nvca-operator" using context "k3d-ncp-local" should complete rollout within "10m" + + Then NVCFBackend "ncp-local" in namespace "nvca-operator" using context "k3d-ncp-local" should report agent status "healthy" within "10m" + + Rule: A function deployment produces SIS events + + Background: + Given these environment variables are set: + | name | + | NGC_API_KEY | + | SAMPLE_NGC_ORG | + | SAMPLE_NGC_TEAM | + | REPO_ROOT | + And I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml" + + # This scenario depends on the earlier control-plane install and NVCA + # registration scenarios in the same feature run. + @compute-plane @function-lifecycle + Scenario: A function deployment produces SIS events that can be read back + When I successfully run command: + """ + /bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z' + """ + And I export command output to environment variable "BDD_EL_KEY_EXPIRES" + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "POST" request to "http://api-keys.localhost:8080/v1/keys" with body: + """ + { + "description": "bdd-event-ledger-lifecycle", + "expires_at": "${BDD_EL_KEY_EXPIRES}", + "authorizations": { + "policies": [ + { + "aud": "nvidia-event-ledger-ncp-service-id-ckozoh6f", + "auds": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"], + "product": "nv-cloud-functions", + "resources": [{"id": "*", "type": "account-functions"}], + "scopes": ["fnds:getEvents", "fnds:getStats"] + } + ] + }, + "audience_service_ids": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"] + } + """ + Then the HTTP status should be 200 + When I export the HTTP response JSON field "value" to environment variable "BDD_EL_READ_KEY" + And I export the HTTP response JSON field "id" to environment variable "BDD_EL_READ_KEY_ID" + + When I successfully create function "bdd-event-ledger" from image "nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/load_tester_supreme:0.0.8" with CLI options: + | option | value | + | --inference-url | /echo | + | --inference-port | 8000 | + | --health-uri | /health | + | --health-port | 8000 | + | --health-timeout | PT30S | + And I successfully deploy the function selected by NVCF CLI with options: + | option | value | + | --gpu | H100 | + | --instance-type | NCP.GPU.H100_1x | + | --backend | ncp-local | + | --regions | us-west-1 | + | --min-instances | 1 | + | --max-instances | 1 | + | --timeout | 900 | + And I export the function selected by NVCF CLI to environment variables "BDD_EL_FUNCTION_ID" and "BDD_EL_VERSION_ID" + + # Events are stored per context, and stats list every context in the + # namespace with its latest event. The NVCA collector, when enabled, adds + # contexts of its own that sort before SIS's, so find the SIS context with + # an event filter, then read its events. + + # SIS writes InstanceReady once the instance is running. + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I poll "GET" "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats" for up to 300 seconds until the HTTP response body contains "InstanceReady" + Then the HTTP status should be 200 + And the HTTP response JSON field "namespace" should equal "${BDD_EL_VERSION_ID}" + And the HTTP response JSON field "summary.by_event.InstanceReady" should be at least 1 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats?eventFilter=InstanceReady" + Then the HTTP status should be 200 + When I export the HTTP response JSON field "contexts.0.context" to environment variable "BDD_EL_CONTEXT" + And I successfully run command: + """ + /bin/bash -c 'echo "${BDD_EL_CONTEXT}" | tr "," "&"' + """ + And I export command output to environment variable "BDD_EL_CONTEXT_QUERY" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/events?${BDD_EL_CONTEXT_QUERY}" + Then the HTTP status should be 200 + And the HTTP response body should contain "InstanceReady" + And the HTTP response body should contain "nvidia-spot" + And the HTTP response body should contain "${BDD_EL_FUNCTION_ID}" + And the HTTP response body should not contain "InstanceDestroyed" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats?eventFilter=NoSuchEvent" + Then the HTTP status should be 200 + And the HTTP response body should not contain "InstanceReady" + + # Undeploy removes the instance and SIS then reports it destroyed. Stats keep + # only the latest event per context, while events keep the whole history. + When I successfully undeploy the function selected by NVCF CLI + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I poll "GET" "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats" for up to 180 seconds until the HTTP response body contains "InstanceDestroyed" + Then the HTTP status should be 200 + And the HTTP response JSON field "summary.by_event.InstanceDestroyed" should be at least 1 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/events?${BDD_EL_CONTEXT_QUERY}" + Then the HTTP status should be 200 + And the HTTP response body should contain "InstanceReady" + And the HTTP response body should contain "InstanceDestroyed" + + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" + Then the HTTP status should be 204 diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 4bdc52d62a..ec239b9bcd 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -2666,8 +2666,9 @@ func TestObservabilityDisabled(t *testing.T) { } // TestEventLedger is the live entry point for the event-ledger feature. It -// runs against an already-running single-cluster ncp-local stack and installs -// nothing. Skipped under -short. +// brings up a single-cluster ncp-local cluster, installs the stack with the +// event ledger enabled, registers the cluster and installs NVCA, then checks +// the ledger. Skipped under -short. func TestEventLedger(t *testing.T) { if testing.Short() { t.Skip("live run skipped under -short") @@ -2675,14 +2676,15 @@ func TestEventLedger(t *testing.T) { runLiveFeature(t, "event-ledger.feature") } -// TestEventLedgerReadOnly runs the event-ledger feature without the function -// lifecycle scenario, so it needs no GPU capacity or image pull access. +// TestEventLedgerControlPlane runs the event-ledger feature without the +// compute-plane scenarios (NVCA registration and the function deployment), so +// it needs no GPU capacity or image pull access. It still installs the stack. // Skipped under -short. -func TestEventLedgerReadOnly(t *testing.T) { +func TestEventLedgerControlPlane(t *testing.T) { if testing.Short() { t.Skip("live run skipped under -short") } - runLiveFeatureTags(t, "event-ledger.feature", "~@function-lifecycle") + runLiveFeatureTags(t, "event-ledger.feature", "~@compute-plane") } // TestMultiClusterHelmfile is the live entry point for the @@ -3038,6 +3040,21 @@ func (f *eventLedgerHTTPFake) events(namespace string, params url.Values) harnes return harness.HTTPResponse{Status: 200, Body: string(body)} } +func eventLedgerHelmListJSON() string { + return `[ +{"name":"nats","namespace":"nats-system","status":"deployed"}, +{"name":"cert-manager","namespace":"cert-manager","status":"deployed"}, +{"name":"openbao-server","namespace":"vault-system","status":"deployed"}, +{"name":"cassandra","namespace":"cassandra-system","status":"deployed"}, +{"name":"api-keys","namespace":"api-keys","status":"deployed"}, +{"name":"sis","namespace":"sis","status":"deployed"}, +{"name":"api","namespace":"nvcf","status":"deployed"}, +{"name":"event-ledger","namespace":"nvcf","status":"deployed"}, +{"name":"ingress","namespace":"envoy-gateway-system","status":"deployed"}, +{"name":"nvca-operator","namespace":"nvca-operator","status":"deployed"} +]` +} + func TestEventLedgerFeatureFileWiresToSteps(t *testing.T) { const selectedFunctionStatusCommand = `/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml status --json` home := t.TempDir() @@ -3052,6 +3069,12 @@ func TestEventLedgerFeatureFileWiresToSteps(t *testing.T) { t.Setenv("REPO_ROOT", "/repo-root-placeholder") destroyed := false runner := newFakeRunner(map[string]harness.Result{ + // Conflict precheck: the multi-cluster control plane must be absent. + "k3d cluster get ncp-local-cp": {ExitCode: 1}, + "helm list --all-namespaces --kube-context k3d-ncp-local -o json": { + ExitCode: 0, + Stdout: eventLedgerHelmListJSON(), + }, selectedFunctionStatusCommand: { ExitCode: 0, Stdout: `{"currentFunction":{"hasFunction":true,"functionId":"function-1","versionId":"version-1"}}`, @@ -3079,6 +3102,11 @@ func TestEventLedgerFeatureFileWiresToSteps(t *testing.T) { suite := newWiringSuite(t, runner) httpFake := &eventLedgerHTTPFake{destroyed: &destroyed} suite.HTTP = httpFake + seedHelmfileLocalBDDFixture(t, suite.Config.RepoRoot) + seedComputePlaneLocalBDDFixture(t, suite.Config.RepoRoot) + seedStackSecretsTemplate(t, suite.Config.RepoRoot) + writeProfileHandoffArtifact(t, suite.Config.RepoRoot) + writeHelmfileRegisterValues(t, suite.Config.RepoRoot) sc := steps.NewScenarioContext(suite) featurePath := mustResolveFeaturePath(t, "event-ledger.feature") @@ -3099,6 +3127,16 @@ func TestEventLedgerFeatureFileWiresToSteps(t *testing.T) { t.Fatalf("godog suite status = %d\n%s", status, out.String()) } runs := suite.Runner.(*fakeRunner).runs + for _, command := range []string{ + "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-event-ledger", + "make -C deploy/stacks/nvcf-compute-plane register-cluster CLUSTER_NAME=ncp-local", + "make -C deploy/stacks/nvcf-compute-plane install CLUSTER_NAME=ncp-local HELMFILE_ENV=local-bdd-event-ledger", + "nvcf-cli-local.yaml init", + } { + if !commandRanThatContains(runs, command) { + t.Fatalf("install flow command was never invoked: %s", command) + } + } if !commandRanThatContains(runs, " function deploy create ") { t.Fatal("function deploy command was never invoked") } From 1beeb83df1969ec7a0a26175686802cd6668e2d9 Mon Sep 17 00:00:00 2001 From: Shelley Shen Date: Sun, 11 Oct 2026 13:11:02 -0700 Subject: [PATCH 10/11] test(bdd): drop the API Keys startup probe override from the local fixture The override only helps a host that starts API Keys slowly, and the fixture is shared by other features. Keep the shared fixture at the stack defaults. A host that needs a longer startup window sets apikeys.startupProbe.failureThreshold in its own environment. This reverts commit 1b4cef26e. Signed-off-by: Shelley Shen --- tests/bdd/fixtures/self-managed-local-bdd.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/bdd/fixtures/self-managed-local-bdd.yaml b/tests/bdd/fixtures/self-managed-local-bdd.yaml index c75eac8c9c..9e733e4f1b 100644 --- a/tests/bdd/fixtures/self-managed-local-bdd.yaml +++ b/tests/bdd/fixtures/self-managed-local-bdd.yaml @@ -101,8 +101,3 @@ ingress: enabled: true ess: enabled: true - -# API Keys can take longer than its default 3 minute startup window on a slow host. -apikeys: - startupProbe: - failureThreshold: 120 From bcf775ccb8e3bba4801eeca0771d701c9b415186 Mon Sep 17 00:00:00 2001 From: Shelley Shen Date: Sun, 11 Oct 2026 20:56:19 -0700 Subject: [PATCH 11/11] test(bdd): clean up event-ledger keys and functions when a scenario ends Add scenario teardown steps that queue a request or a function undeploy and run them in the After hook, so cleanup also happens when a step fails. The event-ledger feature now deletes the API keys it creates and undeploys the lifecycle function this way, expecting status 204 for each key delete. --- tests/bdd/PLAN.md | 11 +- tests/bdd/features/event-ledger.feature | 51 +++---- tests/bdd/godog_test.go | 73 ++++++++++ tests/bdd/steps/context.go | 19 +++ tests/bdd/steps/nvcf_cli_steps.go | 6 +- tests/bdd/steps/teardown_steps.go | 114 +++++++++++++++ tests/bdd/steps/teardown_steps_test.go | 175 ++++++++++++++++++++++++ 7 files changed, 422 insertions(+), 27 deletions(-) create mode 100644 tests/bdd/steps/teardown_steps.go create mode 100644 tests/bdd/steps/teardown_steps_test.go diff --git a/tests/bdd/PLAN.md b/tests/bdd/PLAN.md index 61f5870790..554735c071 100644 --- a/tests/bdd/PLAN.md +++ b/tests/bdd/PLAN.md @@ -174,7 +174,7 @@ original order. Repeated options and empty values are preserved. | `When I successfully invoke the function selected by NVCF CLI over plaintext gRPC service {string} method {string} with timeout {string} seconds and poll duration {string} seconds:` (JSON docstring) | Runs `function invoke --grpc --grpc-plaintext` with the visible service, method, request, timeout, and poll duration. | | `When I successfully invoke model {string} at {string} with timeout {string} seconds:` (JSON docstring) | Runs `function invoke` with the visible model, inference URL, exact request body, and timeout. | | `When I successfully invoke the function selected by NVCF CLI through Vanity Gateway host {string} path {string} with timeout {string} seconds:` (JSON docstring) | Sends an exact-host HTTP request through the local Envoy listener with the saved function API key passed over sensitive stdin, never argv or command logs. | -| `When I successfully undeploy the function selected by NVCF CLI` | Runs `function delete --deployment-only`. Function selection remains owned by CLI state. | +| `When I successfully undeploy the function selected by NVCF CLI` | Runs `function delete --deployment-only`. Function selection remains owned by CLI state. A successful run is remembered, so a queued teardown undeploy skips it. | ### HTTP requests (Given / When / Then) @@ -195,6 +195,15 @@ These steps use the in-process HTTP client (`harness.HTTPClient`), not the comma | `Then the HTTP response JSON field {string} should be at least {int}` | Dotted path parsed as a number and compared to a minimum. | | `When I export the HTTP response JSON field {string} to environment variable {string}` | Exports the field under the named env var. Fails when the field is missing or empty. Snapshotted by the env Ledger; restored at suite teardown. The value is never echoed. | +### Scenario teardown (Given / When) + +Cleanup written as a later step is skipped when an earlier step fails, because the live runner stops at the first failed step. These steps queue cleanup that runs after the scenario ends instead, whether it passed or failed. Queued actions run in reverse order through a Godog After hook, a failing action does not stop the rest, and every failure is reported by failing the scenario. The queue is cleared before each scenario. + +| Step | Notes | +|------|-------| +| `When I send a {string} request to {string} at scenario teardown expecting status {int}` | Queues the request instead of sending it now. It takes the queued headers like the send step, so declare a credential just before it. `${VAR}` expansion applies to the URL when the step runs, so the variable must already be exported. Any status other than the expected one fails the scenario. | +| `When the function selected by NVCF CLI is undeployed at scenario teardown` | Queues `function delete --deployment-only` for the selected CLI config. It does nothing when the scenario already undeployed the function with the undeploy step. A "Deployment not found" result counts as done, and any other failure fails the scenario. | + ### Assertions (Then / And) | Step | Notes | diff --git a/tests/bdd/features/event-ledger.feature b/tests/bdd/features/event-ledger.feature index 4eda2c434c..24e5b33f5c 100644 --- a/tests/bdd/features/event-ledger.feature +++ b/tests/bdd/features/event-ledger.feature @@ -177,6 +177,12 @@ Feature: Install a local single-cluster NVCF stack and check the event ledger Then the HTTP status should be 200 When I export the HTTP response JSON field "value" to environment variable "BDD_EL_READ_KEY" And I export the HTTP response JSON field "id" to environment variable "BDD_EL_READ_KEY_ID" + # Delete the key when the scenario ends, even if it fails. + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + And I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" at scenario teardown expecting status 204 Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/stats" @@ -201,12 +207,6 @@ Feature: Install a local single-cluster NVCF stack and check the event ledger When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/stats?view=bogus" Then the HTTP status should be 400 - Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " - And the HTTP request header "Key-Issuer-Service" is "nvcf-api" - And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" - And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" - When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" - Then the HTTP status should be 204 # Write access needs an OpenBao JWT with the FnDS write scope, so a # read-scoped api key must be rejected on the write route. @@ -243,6 +243,12 @@ Feature: Install a local single-cluster NVCF stack and check the event ledger Then the HTTP status should be 200 When I export the HTTP response JSON field "value" to environment variable "BDD_EL_READ_KEY" And I export the HTTP response JSON field "id" to environment variable "BDD_EL_READ_KEY_ID" + # Delete the key when the scenario ends, even if it fails. + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + And I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" at scenario teardown expecting status 204 Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" And the HTTP request header "Content-Type" is "application/cloudevents+json" @@ -259,12 +265,6 @@ Feature: Install a local single-cluster NVCF stack and check the event ledger When I send a "POST" request to "http://events.localhost:8080/v3/ledger/k8s-events" Then the HTTP status should be 403 - Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " - And the HTTP request header "Key-Issuer-Service" is "nvcf-api" - And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" - And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" - When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" - Then the HTTP status should be 204 # The positive control for the scope checks: a key with only the write scope # may write, which shows the rejections above are specific and not an outage, @@ -302,6 +302,12 @@ Feature: Install a local single-cluster NVCF stack and check the event ledger Then the HTTP status should be 200 When I export the HTTP response JSON field "value" to environment variable "BDD_EL_WRITE_KEY" And I export the HTTP response JSON field "id" to environment variable "BDD_EL_WRITE_KEY_ID" + # Delete the key when the scenario ends, even if it fails. + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + And I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_WRITE_KEY_ID}" at scenario teardown expecting status 204 Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" And the HTTP request header "Content-Type" is "application/cloudevents+json" @@ -329,12 +335,6 @@ Feature: Install a local single-cluster NVCF stack and check the event ledger When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-write-scope/stats" Then the HTTP status should be 403 - Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " - And the HTTP request header "Key-Issuer-Service" is "nvcf-api" - And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" - And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" - When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_WRITE_KEY_ID}" - Then the HTTP status should be 204 Rule: Helmfile installs NVCA on the same local cluster after registration @@ -424,6 +424,12 @@ Feature: Install a local single-cluster NVCF stack and check the event ledger Then the HTTP status should be 200 When I export the HTTP response JSON field "value" to environment variable "BDD_EL_READ_KEY" And I export the HTTP response JSON field "id" to environment variable "BDD_EL_READ_KEY_ID" + # Delete the key when the scenario ends, even if it fails. + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + And I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" at scenario teardown expecting status 204 When I successfully create function "bdd-event-ledger" from image "nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/load_tester_supreme:0.0.8" with CLI options: | option | value | @@ -441,6 +447,8 @@ Feature: Install a local single-cluster NVCF stack and check the event ledger | --min-instances | 1 | | --max-instances | 1 | | --timeout | 900 | + # Undeploy when the scenario ends if it has not already, even if it fails. + And the function selected by NVCF CLI is undeployed at scenario teardown And I export the function selected by NVCF CLI to environment variables "BDD_EL_FUNCTION_ID" and "BDD_EL_VERSION_ID" # Events are stored per context, and stats list every context in the @@ -491,10 +499,3 @@ Feature: Install a local single-cluster NVCF stack and check the event ledger Then the HTTP status should be 200 And the HTTP response body should contain "InstanceReady" And the HTTP response body should contain "InstanceDestroyed" - - Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " - And the HTTP request header "Key-Issuer-Service" is "nvcf-api" - And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" - And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" - When I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" - Then the HTTP status should be 204 diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index ec239b9bcd..d67301006a 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -23,6 +23,7 @@ import ( "context" "encoding/json" "io" + "net/http" "net/url" "os" "path/filepath" @@ -3055,6 +3056,78 @@ func eventLedgerHelmListJSON() string { ]` } +// recordingHTTP is an HTTPClient that records requests and answers every one +// with a fixed status. +type recordingHTTP struct { + status int + requests []string + headers []map[string]string +} + +func (r *recordingHTTP) Do(_ context.Context, method, url string, headers map[string]string, _ string) (harness.HTTPResponse, error) { + r.requests = append(r.requests, method+" "+url) + r.headers = append(r.headers, headers) + return harness.HTTPResponse{Status: r.status}, nil +} + +func runTeardownFeature(t *testing.T, body string, httpClient harness.HTTPClient) int { + t.Helper() + feature := filepath.Join(t.TempDir(), "teardown.feature") + if err := os.WriteFile(feature, []byte(body), 0o644); err != nil { + t.Fatalf("write feature: %v", err) + } + suite := newWiringSuite(t, newFakeRunner(nil)) + suite.HTTP = httpClient + sc := steps.NewScenarioContext(suite) + return godog.TestSuite{ + Name: "teardown-wiring", + ScenarioInitializer: func(ctx *godog.ScenarioContext) { + steps.RegisterAll(ctx, sc) + }, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{feature}, + Strict: true, + StopOnFailure: true, + Output: io.Discard, + }, + }.Run() +} + +// TestScenarioTeardownRunsAfterAFailedStep guards the live runner stopping at +// the first failed step: cleanup written as a later step would be skipped, a +// teardown must not be. +func TestScenarioTeardownRunsAfterAFailedStep(t *testing.T) { + recorder := &recordingHTTP{status: http.StatusNoContent} + status := runTeardownFeature(t, `Feature: teardown + Scenario: cleanup survives a failed step + Given the HTTP request header "Key-Owner-Id" is "owner" + And I send a "DELETE" request to "http://api-keys/v1/keys/key-1" at scenario teardown expecting status 204 + Then the HTTP status should be 200 +`, recorder) + if status == 0 { + t.Fatal("the scenario should fail on its assertion") + } + if len(recorder.requests) != 1 || recorder.requests[0] != "DELETE http://api-keys/v1/keys/key-1" { + t.Fatalf("teardown requests = %v", recorder.requests) + } + if recorder.headers[0]["Key-Owner-Id"] != "owner" { + t.Fatalf("teardown headers = %v", recorder.headers[0]) + } +} + +func TestScenarioTeardownFailureFailsAnOtherwisePassingScenario(t *testing.T) { + recorder := &recordingHTTP{status: http.StatusInternalServerError} + status := runTeardownFeature(t, `Feature: teardown + Scenario: cleanup that fails is reported + When I send a "GET" request to "http://example/ok" + And I send a "DELETE" request to "http://api-keys/v1/keys/key-1" at scenario teardown expecting status 204 +`, recorder) + if status == 0 { + t.Fatal("a failed teardown should fail the scenario") + } +} + func TestEventLedgerFeatureFileWiresToSteps(t *testing.T) { const selectedFunctionStatusCommand = `/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml status --json` home := t.TempDir() diff --git a/tests/bdd/steps/context.go b/tests/bdd/steps/context.go index 4dc723be70..d756a7d3ec 100644 --- a/tests/bdd/steps/context.go +++ b/tests/bdd/steps/context.go @@ -56,6 +56,17 @@ type ScenarioContext struct { Manifests map[string]string HTTPHeaders map[string]string LastHTTP *harness.HTTPResponse + + // FunctionUndeployed is set once the scenario has undeployed the selected + // function, so the teardown does not undeploy it again. + FunctionUndeployed bool + teardown []teardownAction +} + +// teardownAction is cleanup queued for the end of the scenario. +type teardownAction struct { + description string + run func(context.Context) error } // NewScenarioContext wraps suite in a fresh per-scenario state. The @@ -76,8 +87,15 @@ func RegisterAll(ctx *godog.ScenarioContext, sc *ScenarioContext) { sc.Manifests = nil sc.HTTPHeaders = nil sc.LastHTTP = nil + sc.FunctionUndeployed = false + sc.teardown = nil return c, nil }) + // Godog runs After hooks for failed scenarios too, which is what lets + // cleanup survive the live runner stopping at the first failed step. + ctx.After(func(c context.Context, _ *godog.Scenario, _ error) (context.Context, error) { + return c, sc.runTeardown(c) + }) // Godog's default pretty formatter buffers the scenario block until // every step finishes, which makes hangs invisible during a live // run. Echo each step's text to stderr the moment it starts so the @@ -106,6 +124,7 @@ func RegisterAll(ctx *godog.ScenarioContext, sc *ScenarioContext) { registerManifestSteps(ctx, sc) registerCommandSteps(ctx, sc) registerHTTPSteps(ctx, sc) + registerTeardownSteps(ctx, sc) registerNVCFCLISteps(ctx, sc) registerRegistrationSteps(ctx, sc) registerAssertionSteps(ctx, sc) diff --git a/tests/bdd/steps/nvcf_cli_steps.go b/tests/bdd/steps/nvcf_cli_steps.go index e34efea48c..b7285bc97a 100644 --- a/tests/bdd/steps/nvcf_cli_steps.go +++ b/tests/bdd/steps/nvcf_cli_steps.go @@ -258,7 +258,11 @@ func isTransientVanityRouteNotFound(stderr string) bool { } func (sc *ScenarioContext) iSuccessfullyUndeploySelectedFunction(ctx context.Context) error { - return sc.runNVCFCLI(ctx, "function", "delete", "--deployment-only") + if err := sc.runNVCFCLI(ctx, "function", "delete", "--deployment-only"); err != nil { + return err + } + sc.FunctionUndeployed = true + return nil } func (sc *ScenarioContext) runNVCFCLIWithOptions( diff --git a/tests/bdd/steps/teardown_steps.go b/tests/bdd/steps/teardown_steps.go new file mode 100644 index 0000000000..3c070ceddf --- /dev/null +++ b/tests/bdd/steps/teardown_steps.go @@ -0,0 +1,114 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package steps + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "github.com/cucumber/godog" + + "nvcf-bdd/dsl" +) + +// functionDeploymentNotFound is how the NVCF API reports a function with no +// deployment, which is the expected state when a scenario already undeployed it. +const functionDeploymentNotFound = "Deployment not found" + +// registerTeardownSteps hooks the steps that register cleanup to run after the +// scenario ends, whether it passed or failed. The live runner stops at the +// first failed step, so cleanup written as a later step would never run. +func registerTeardownSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { + ctx.Step(`^I send an? "([A-Z]+)" request to "([^"]*)" at scenario teardown expecting status (\d+)$`, sc.iSendHTTPRequestAtTeardown) + ctx.Step(`^the function selected by NVCF CLI is undeployed at scenario teardown$`, sc.theSelectedFunctionIsUndeployedAtTeardown) +} + +// addTeardown queues cleanup for the end of the current scenario. +func (sc *ScenarioContext) addTeardown(description string, action func(context.Context) error) { + sc.teardown = append(sc.teardown, teardownAction{description: description, run: action}) +} + +// iSendHTTPRequestAtTeardown queues a request for the end of the scenario. It +// takes the headers queued so far, like the send step does, so a credential is +// declared next to the request that needs it. The URL is resolved now, so any +// variable it uses must already be exported. Any status other than the +// expected one fails the scenario. +func (sc *ScenarioContext) iSendHTTPRequestAtTeardown(method, url string, wantStatus int) error { + headers := sc.HTTPHeaders + sc.HTTPHeaders = nil + resolved := dsl.Interpolate(url) + sc.addTeardown(method+" "+resolved, func(ctx context.Context) error { + resp, err := sc.httpClient().Do(ctx, method, resolved, headers, "") + if err != nil { + return err + } + if resp.Status != wantStatus { + return fmt.Errorf("status %d, want %d: %s", resp.Status, wantStatus, truncateBody(resp.Body)) + } + return nil + }) + return nil +} + +// theSelectedFunctionIsUndeployedAtTeardown queues an undeploy of the function +// the NVCF CLI has selected. It does nothing when the scenario already +// undeployed the function, and treats a function with no deployment as done. +func (sc *ScenarioContext) theSelectedFunctionIsUndeployedAtTeardown() error { + if sc.NVCFCLIConfig == "" { + return fmt.Errorf("no NVCF CLI config selected; use 'I use NVCF CLI config' first") + } + command := dsl.BuildCommand(sc.nvcfCLICommandArgs("function", "delete", "--deployment-only")...) + sc.addTeardown("undeploy the selected function", func(ctx context.Context) error { + if sc.FunctionUndeployed { + return nil + } + result, err := sc.Suite.Runner.Run(ctx, command) + if err == nil && result.ExitCode == 0 { + return nil + } + output := combinedOutput(result) + if strings.Contains(output, functionDeploymentNotFound) { + return nil + } + if err != nil { + return fmt.Errorf("exit code %d: %w", result.ExitCode, err) + } + return fmt.Errorf("exit code %d", result.ExitCode) + }) + return nil +} + +// runTeardown runs the queued cleanup in reverse order, so a resource is +// released before the one it depends on. A failing action does not stop the +// rest. The queue is cleared and every failure is reported together. +func (sc *ScenarioContext) runTeardown(ctx context.Context) error { + actions := sc.teardown + sc.teardown = nil + var failures []error + for i := len(actions) - 1; i >= 0; i-- { + action := actions[i] + fmt.Fprintf(os.Stderr, ">>> teardown: %s\n", action.description) + if err := action.run(ctx); err != nil { + failures = append(failures, fmt.Errorf("teardown %q: %w", action.description, err)) + } + } + return errors.Join(failures...) +} diff --git a/tests/bdd/steps/teardown_steps_test.go b/tests/bdd/steps/teardown_steps_test.go new file mode 100644 index 0000000000..e097b9e666 --- /dev/null +++ b/tests/bdd/steps/teardown_steps_test.go @@ -0,0 +1,175 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package steps + +import ( + "context" + "errors" + "strings" + "testing" + + "nvcf-bdd/harness" +) + +func TestHTTPTeardownSendsRegisteredRequestWithItsHeaders(t *testing.T) { + t.Setenv("BDD_TEARDOWN_KEY_ID", "key-1") + sc, fake := newHTTPScenario(t, harness.HTTPResponse{Status: 204}) + if err := sc.theHTTPRequestHeaderIs("Key-Owner-Id", "owner"); err != nil { + t.Fatal(err) + } + if err := sc.iSendHTTPRequestAtTeardown("DELETE", "http://api-keys/v1/keys/${BDD_TEARDOWN_KEY_ID}", 204); err != nil { + t.Fatal(err) + } + if fake.calls != 0 { + t.Fatalf("request sent at registration: %d calls", fake.calls) + } + if len(sc.HTTPHeaders) != 0 { + t.Fatalf("pending headers not consumed: %v", sc.HTTPHeaders) + } + + if err := sc.runTeardown(context.Background()); err != nil { + t.Fatalf("teardown: %v", err) + } + if fake.method != "DELETE" || fake.url != "http://api-keys/v1/keys/key-1" || fake.headers["Key-Owner-Id"] != "owner" { + t.Fatalf("sent %s %s headers=%v", fake.method, fake.url, fake.headers) + } +} + +func TestHTTPTeardownReportsFailure(t *testing.T) { + sc, _ := newHTTPScenario(t, harness.HTTPResponse{Status: 500, Body: "boom"}) + if err := sc.iSendHTTPRequestAtTeardown("DELETE", "http://x/", 204); err != nil { + t.Fatal(err) + } + err := sc.runTeardown(context.Background()) + if err == nil || !strings.Contains(err.Error(), "status 500, want 204") { + t.Fatalf("expected status failure, got %v", err) + } +} + +func TestHTTPTeardownRequiresTheExactStatus(t *testing.T) { + sc, _ := newHTTPScenario(t, harness.HTTPResponse{Status: 200}) + if err := sc.iSendHTTPRequestAtTeardown("DELETE", "http://x/", 204); err != nil { + t.Fatal(err) + } + if err := sc.runTeardown(context.Background()); err == nil || !strings.Contains(err.Error(), "status 200, want 204") { + t.Fatalf("a 200 must not satisfy an expected 204, got %v", err) + } +} + +func TestTeardownRunsInReverseOrderAndContinuesAfterFailure(t *testing.T) { + sc, _ := newScenarioContext(t) + var order []string + for _, name := range []string{"first", "second", "third"} { + sc.addTeardown(name, func(context.Context) error { + order = append(order, name) + if name == "second" { + return errors.New("second failed") + } + return nil + }) + } + err := sc.runTeardown(context.Background()) + if strings.Join(order, ",") != "third,second,first" { + t.Fatalf("order = %v", order) + } + if err == nil || !strings.Contains(err.Error(), "second failed") { + t.Fatalf("expected the second failure, got %v", err) + } + if len(sc.teardown) != 0 { + t.Fatal("queue not cleared") + } + if err := sc.runTeardown(context.Background()); err != nil { + t.Fatalf("second run should be a no-op: %v", err) + } +} + +func TestUndeployTeardownRequiresSelectedConfig(t *testing.T) { + sc, _ := newScenarioContext(t) + if err := sc.theSelectedFunctionIsUndeployedAtTeardown(); err == nil { + t.Fatal("expected an error without an NVCF CLI config") + } +} + +func TestUndeployTeardownRunsTheCLIOnce(t *testing.T) { + t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") + sc, fake := newScenarioContext(t) + sc.NVCFCLIConfig = "/cfg.yaml" + if err := sc.theSelectedFunctionIsUndeployedAtTeardown(); err != nil { + t.Fatal(err) + } + if len(fake.runs) != 0 { + t.Fatal("undeploy ran at registration") + } + if err := sc.runTeardown(context.Background()); err != nil { + t.Fatal(err) + } + if len(fake.runs) != 1 || !strings.Contains(fake.runs[0].command, "function delete --deployment-only") || + !strings.Contains(fake.runs[0].command, "--config /cfg.yaml") { + t.Fatalf("runs = %+v", fake.runs) + } +} + +func TestUndeployTeardownSkipsAfterTheScenarioUndeployed(t *testing.T) { + t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") + sc, fake := newScenarioContext(t) + sc.NVCFCLIConfig = "/cfg.yaml" + if err := sc.theSelectedFunctionIsUndeployedAtTeardown(); err != nil { + t.Fatal(err) + } + if err := sc.iSuccessfullyUndeploySelectedFunction(context.Background()); err != nil { + t.Fatal(err) + } + if !sc.FunctionUndeployed { + t.Fatal("successful undeploy not remembered") + } + runsBefore := len(fake.runs) + if err := sc.runTeardown(context.Background()); err != nil { + t.Fatal(err) + } + if len(fake.runs) != runsBefore { + t.Fatalf("teardown undeployed again: %+v", fake.runs[runsBefore:]) + } +} + +func TestUndeployTeardownToleratesAFunctionWithNoDeployment(t *testing.T) { + t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") + sc, fake := newScenarioContext(t) + sc.NVCFCLIConfig = "/cfg.yaml" + fake.result = harness.Result{ExitCode: 1, Stderr: `Error: API error 404: {"detail":"Deployment not found"}`} + fake.err = errors.New("exit status 1") + if err := sc.theSelectedFunctionIsUndeployedAtTeardown(); err != nil { + t.Fatal(err) + } + if err := sc.runTeardown(context.Background()); err != nil { + t.Fatalf("not-found should count as already undeployed: %v", err) + } +} + +func TestUndeployTeardownReportsOtherFailures(t *testing.T) { + t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") + sc, fake := newScenarioContext(t) + sc.NVCFCLIConfig = "/cfg.yaml" + fake.result = harness.Result{ExitCode: 1, Stderr: "Error: connection refused"} + fake.err = errors.New("exit status 1") + if err := sc.theSelectedFunctionIsUndeployedAtTeardown(); err != nil { + t.Fatal(err) + } + if err := sc.runTeardown(context.Background()); err == nil { + t.Fatal("expected the failure to be reported") + } +}