diff --git a/deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl index a5efe9bee1..4bc9273942 100644 --- a/deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl @@ -285,7 +285,7 @@ releases: - name: event-ledger chart: nvcf/helm-nvcf-event-ledger - version: 0.5.0 + version: 0.6.0 namespace: nvcf condition: addons.eventLedger.enabled values: diff --git a/tests/bdd/PLAN.md b/tests/bdd/PLAN.md index 2b5caed9aa..554735c071 100644 --- a/tests/bdd/PLAN.md +++ b/tests/bdd/PLAN.md @@ -174,7 +174,35 @@ original order. Repeated options and empty values are preserved. | `When I successfully invoke the function selected by NVCF CLI over plaintext gRPC service {string} method {string} with timeout {string} seconds and poll duration {string} seconds:` (JSON docstring) | Runs `function invoke --grpc --grpc-plaintext` with the visible service, method, request, timeout, and poll duration. | | `When I successfully invoke model {string} at {string} with timeout {string} seconds:` (JSON docstring) | Runs `function invoke` with the visible model, inference URL, exact request body, and timeout. | | `When I successfully invoke the function selected by NVCF CLI through Vanity Gateway host {string} path {string} with timeout {string} seconds:` (JSON docstring) | Sends an exact-host HTTP request through the local Envoy listener with the saved function API key passed over sensitive stdin, never argv or command logs. | -| `When I successfully undeploy the function selected by NVCF CLI` | Runs `function delete --deployment-only`. Function selection remains owned by CLI state. | +| `When I successfully undeploy the function selected by NVCF CLI` | Runs `function delete --deployment-only`. Function selection remains owned by CLI state. A successful run is remembered, so a queued teardown undeploy skips it. | + +### HTTP requests (Given / When / Then) + +These steps use the in-process HTTP client (`harness.HTTPClient`), not the command runner, so headers never reach `out//logs`. Redirects are not followed. Pending headers apply to the next send or poll step only. + +| Step | Notes | +|------|-------| +| `Given the HTTP request header {string} is {string}` | Queues one request header for the next request. `${VAR}` expansion applies to the value. | +| `Given the HTTP request header {string} is the NVCF CLI admin token with prefix {string}` | Queues a header whose value is the prefix plus the admin token stored by `nvcf-cli init` in the state file of the selected NVCF CLI config. The token never enters argv, env, or logs. | +| `When I send a {string} request to {string}` | Sends the request with the queued headers and records status and body. `${VAR}` expansion applies to the URL. A non-2xx status is not a step failure. | +| `When I send a {string} request to {string} with body:` (docstring) | As above with the interpolated docstring as the request body. | +| `When I poll {string} {string} for up to {int} seconds until the HTTP response body contains {string}` | Repeats the request every 5 seconds until the body contains the text or the deadline passes. Used for eventually consistent reads. | +| `Then the HTTP status should be {int}` | Last response status. The failure message includes a truncated body. | +| `Then the HTTP status should be {int} or {int}` | Last response status equals either value. | +| `Then the HTTP response body should contain {string}` | Substring match. The interpolated value must be non-empty. | +| `Then the HTTP response body should not contain {string}` | Negative substring match. The interpolated value must be non-empty. | +| `Then the HTTP response JSON field {string} should equal {string}` | Dotted path (object keys, array indexes) compared as text. | +| `Then the HTTP response JSON field {string} should be at least {int}` | Dotted path parsed as a number and compared to a minimum. | +| `When I export the HTTP response JSON field {string} to environment variable {string}` | Exports the field under the named env var. Fails when the field is missing or empty. Snapshotted by the env Ledger; restored at suite teardown. The value is never echoed. | + +### Scenario teardown (Given / When) + +Cleanup written as a later step is skipped when an earlier step fails, because the live runner stops at the first failed step. These steps queue cleanup that runs after the scenario ends instead, whether it passed or failed. Queued actions run in reverse order through a Godog After hook, a failing action does not stop the rest, and every failure is reported by failing the scenario. The queue is cleared before each scenario. + +| Step | Notes | +|------|-------| +| `When I send a {string} request to {string} at scenario teardown expecting status {int}` | Queues the request instead of sending it now. It takes the queued headers like the send step, so declare a credential just before it. `${VAR}` expansion applies to the URL when the step runs, so the variable must already be exported. Any status other than the expected one fails the scenario. | +| `When the function selected by NVCF CLI is undeployed at scenario teardown` | Queues `function delete --deployment-only` for the selected CLI config. It does nothing when the scenario already undeployed the function with the undeploy step. A "Deployment not found" result counts as done, and any other failure fails the scenario. | ### Assertions (Then / And) diff --git a/tests/bdd/dsl/clistate.go b/tests/bdd/dsl/clistate.go new file mode 100644 index 0000000000..92d689a857 --- /dev/null +++ b/tests/bdd/dsl/clistate.go @@ -0,0 +1,52 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dsl + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +// CLIStatePath returns the nvcf-cli state file for a --config path, mirroring +// the CLI: ~/.nvcf-cli..state. +func CLIStatePath(home, configPath string) string { + name := strings.TrimSuffix(filepath.Base(configPath), filepath.Ext(configPath)) + return filepath.Join(home, fmt.Sprintf(".nvcf-cli.%s.state", name)) +} + +// CLIStateToken reads the admin token that `nvcf-cli init` stored in the +// state file at path. It fails when the file or the token is missing. +func CLIStateToken(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read nvcf-cli state: %w", err) + } + var state struct { + Token string `json:"token"` + } + if err := json.Unmarshal(data, &state); err != nil { + return "", fmt.Errorf("parse nvcf-cli state %s: %w", path, err) + } + if state.Token == "" { + return "", fmt.Errorf("nvcf-cli state %s has no admin token; run nvcf-cli init first", path) + } + return state.Token, nil +} diff --git a/tests/bdd/dsl/http_test.go b/tests/bdd/dsl/http_test.go new file mode 100644 index 0000000000..28f055c127 --- /dev/null +++ b/tests/bdd/dsl/http_test.go @@ -0,0 +1,107 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dsl + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +const ledgerEvents = `{"namespace":"ns","summary":{"total_contexts":2,"by_event":{"InstanceReady":2}},"events":[{"event_name":"InstanceReady","source":"nvidia-spot"}]}` + +func TestJSONFieldReadsNestedValues(t *testing.T) { + cases := map[string]string{ + "namespace": "ns", + "summary.total_contexts": "2", + "summary.by_event.InstanceReady": "2", + "events.0.event_name": "InstanceReady", + "events.0.source": "nvidia-spot", + } + for path, want := range cases { + got, err := JSONField(ledgerEvents, path) + if err != nil { + t.Fatalf("%s: %v", path, err) + } + if got != want { + t.Fatalf("%s = %q, want %q", path, got, want) + } + } +} + +func TestJSONFieldReturnsCompactJSONForObjects(t *testing.T) { + got, err := JSONField(ledgerEvents, "summary.by_event") + if err != nil { + t.Fatal(err) + } + if got != `{"InstanceReady":2}` { + t.Fatalf("got %q", got) + } +} + +func TestJSONFieldErrors(t *testing.T) { + for _, path := range []string{"", "missing", "events.5", "events.x", "namespace.child"} { + if _, err := JSONField(ledgerEvents, path); err == nil { + t.Fatalf("path %q: expected error", path) + } + } + if _, err := JSONField("not json", "a"); err == nil { + t.Fatal("expected parse error") + } +} + +func TestJSONFieldAtLeast(t *testing.T) { + if err := JSONFieldAtLeast(ledgerEvents, "summary.by_event.InstanceReady", 1); err != nil { + t.Fatalf("at least 1: %v", err) + } + if err := JSONFieldAtLeast(ledgerEvents, "summary.by_event.InstanceReady", 3); err == nil { + t.Fatal("expected failure for min 3") + } + if err := JSONFieldAtLeast(ledgerEvents, "namespace", 1); err == nil { + t.Fatal("expected failure for non-numeric field") + } +} + +func TestCLIStatePathMirrorsCLI(t *testing.T) { + got := CLIStatePath("/home/u", "/repo/tests/bdd/fixtures/nvcf-cli-local.yaml") + if got != "/home/u/.nvcf-cli.nvcf-cli-local.state" { + t.Fatalf("got %q", got) + } +} + +func TestCLIStateToken(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "state") + if err := os.WriteFile(path, []byte(`{"token":"abc"}`), 0o600); err != nil { + t.Fatal(err) + } + got, err := CLIStateToken(path) + if err != nil || got != "abc" { + t.Fatalf("token = %q, err = %v", got, err) + } + if err := os.WriteFile(path, []byte(`{}`), 0o600); err != nil { + t.Fatal(err) + } + if _, err := CLIStateToken(path); err == nil || !strings.Contains(err.Error(), "nvcf-cli init") { + t.Fatalf("expected missing-token error, got %v", err) + } + if _, err := CLIStateToken(filepath.Join(dir, "absent")); err == nil { + t.Fatal("expected read error") + } +} diff --git a/tests/bdd/dsl/jsonfield.go b/tests/bdd/dsl/jsonfield.go new file mode 100644 index 0000000000..45fb14d0a1 --- /dev/null +++ b/tests/bdd/dsl/jsonfield.go @@ -0,0 +1,100 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dsl + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" +) + +// JSONField reads the dotted path from a JSON document and returns the value +// as text. Object keys are matched by name and array elements by decimal +// index, so "events.0.event_name" reads the first event. Scalars come back as +// their plain text; objects and arrays come back as compact JSON. +func JSONField(raw, path string) (string, error) { + if strings.TrimSpace(path) == "" { + return "", fmt.Errorf("json path must be non-empty") + } + var node any + if err := json.Unmarshal([]byte(raw), &node); err != nil { + return "", fmt.Errorf("parse json: %w", err) + } + for _, part := range strings.Split(path, ".") { + next, err := childNode(node, part) + if err != nil { + return "", fmt.Errorf("json path %q: %w", path, err) + } + node = next + } + return nodeText(node) +} + +// JSONFieldAtLeast reports whether the numeric value at the dotted path is +// greater than or equal to min. +func JSONFieldAtLeast(raw, path string, min float64) error { + text, err := JSONField(raw, path) + if err != nil { + return err + } + value, err := strconv.ParseFloat(text, 64) + if err != nil { + return fmt.Errorf("json path %q is not a number: %q", path, text) + } + if value < min { + return fmt.Errorf("json path %q = %v, want at least %v", path, value, min) + } + return nil +} + +func childNode(node any, part string) (any, error) { + switch typed := node.(type) { + case map[string]any: + child, ok := typed[part] + if !ok { + return nil, fmt.Errorf("key %q not found", part) + } + return child, nil + case []any: + index, err := strconv.Atoi(part) + if err != nil || index < 0 || index >= len(typed) { + return nil, fmt.Errorf("index %q out of range for array of %d", part, len(typed)) + } + return typed[index], nil + default: + return nil, fmt.Errorf("cannot descend into %q: value is not an object or array", part) + } +} + +func nodeText(node any) (string, error) { + switch typed := node.(type) { + case string: + return typed, nil + case nil: + return "null", nil + case map[string]any, []any: + data, err := json.Marshal(typed) + if err != nil { + return "", fmt.Errorf("encode json: %w", err) + } + return string(data), nil + default: + return fmt.Sprint(typed), nil + } +} diff --git a/tests/bdd/features/event-ledger.feature b/tests/bdd/features/event-ledger.feature new file mode 100644 index 0000000000..24e5b33f5c --- /dev/null +++ b/tests/bdd/features/event-ledger.feature @@ -0,0 +1,501 @@ +@event-ledger @ncp-local @single-cluster @helmfile +Feature: Install a local single-cluster NVCF stack and check the event ledger + As a self-managed NVCF operator, + I want to install the stack with the event ledger enabled, + so that I can check it is reachable, protected, and fed by SIS. + + Rule: Helmfile installs the local control plane with the event ledger enabled + + Background: + Given these environment variables are set: + | name | + | NGC_API_KEY | + | SAMPLE_NGC_ORG | + | SAMPLE_NGC_TEAM | + | NVCF_CLI | + | REPO_ROOT | + # The fixture is a copy of deploy/stacks/self-managed/environments/local.yaml, + # which already carries every ncp-local local-mode override. The Background + # only overlays the values that vary per NGC org and pull-secret name. The + # event ledger add-on is on by default in the stack's base environment. + And I prepare Helmfile environment "local-bdd-event-ledger" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd.yaml" with values: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + And I prepare Helmfile environment "local-bdd-event-ledger" for stack "nvcf-compute-plane" from fixture "tests/bdd/fixtures/nvcf-compute-plane-local-bdd.yaml" with values: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-event-ledger-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential + # Conflict precheck: ncp-local-cp's k3d serverlb claims + # 0.0.0.0:8080/8443/10081, NATS on 4222, and the worker callback port + # 10086, overlapping host ports single-cluster ncp-local needs. Fail loudly + # so the operator runs `make -C tools/ncp-local-cluster destroy-multicluster` + # before retrying. `k3d cluster get` exits 1 when absent (k3d v5). + And I run command "k3d cluster get ncp-local-cp" + And the command exit code should be 1 + And a single-cluster ncp-local cluster is running + And the "nvcr-pull-secret" image pull secret exists in namespaces: + | cassandra-system | + | nats-system | + | nvcf | + | api-keys | + | ess | + | sis | + | vault-system | + | nvca-operator | + | cert-manager | + + Scenario: Operator installs the control plane with the event ledger enabled + When I successfully run command "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-event-ledger" + + Then these Helm releases should be deployed using context "k3d-ncp-local": + | name | namespace | + | nats | nats-system | + | cert-manager | cert-manager | + | openbao-server | vault-system | + | cassandra | cassandra-system | + | api-keys | api-keys | + | sis | sis | + | api | nvcf | + | event-ledger | nvcf | + | ingress | envoy-gateway-system | + + # The checks below create API keys, which needs the admin token. + When I successfully run command "${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml init" + + Rule: The event ledger is reachable, protected, and readable + + Background: + Given these environment variables are set: + | name | + | NGC_API_KEY | + | SAMPLE_NGC_ORG | + | SAMPLE_NGC_TEAM | + | REPO_ROOT | + And I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml" + + # These scenarios depend on the earlier control-plane install scenario in the + # same feature run and are not standalone tag targets. + Scenario: Event ledger is running and routed through the shared gateway + Then deployment "event-ledger" in namespace "nvcf" using context "k3d-ncp-local" should complete rollout within "2m" + And these Kubernetes resources should exist in namespace "nvcf" using context "k3d-ncp-local": + | kind | name | + | service | event-ledger | + And these Gateway API routes should be accepted and resolved using context "k3d-ncp-local" within "2m": + | kind | name | namespace | parent | + | HTTPRoute | event-ledger | envoy-gateway-system | shared-gw | + + When I successfully run command "kubectl --context k3d-ncp-local --namespace nvcf get service event-ledger -o jsonpath={.spec.ports[*].name}" + Then the command output should contain "api-port" + + When I send a "GET" request to "http://events.localhost:8080/health" + Then the HTTP status should be 200 + + Scenario: SIS is configured to send events to the event ledger + When I successfully run command "kubectl --context k3d-ncp-local --namespace sis get configmaps -o yaml" + Then the command output should contain all: + | text | + | ICMS_FNDS_MESSAGES_ENABLED: "true" | + | ICMS_FNDS_MESSAGES_V3_ENABLED: "true" | + + # A missing or JWT-shaped invalid credential is rejected with 401. An opaque + # token is not a JWT, so it goes to the api-keys policy evaluator, which + # denies an unknown key with 403. + Scenario: Requests without valid credentials are rejected + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/cloudevents" with body: + """ + {"specversion":"1.0"} + """ + Then the HTTP status should be 401 + + Given the HTTP request header "Authorization" is "Bearer eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ4In0.c2ln" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/cloudevents" + Then the HTTP status should be 401 + + Given the HTTP request header "Authorization" is "Bearer not-a-token" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/k8s-events" + Then the HTTP status should be 403 + + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-auth/stats" + Then the HTTP status should be 401 + + Given the HTTP request header "Authorization" is "Bearer eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ4In0.c2ln" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-auth/events" + Then the HTTP status should be 401 + + Given the HTTP request header "Authorization" is "Bearer nvapi-not-a-real-key" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-auth/events" + Then the HTTP status should be 403 + + When I send a "GET" request to "http://events.localhost:8080/status" + Then the HTTP status should be 401 + + Scenario: V1 and V2 endpoints are disabled + When I send a "POST" request to "http://events.localhost:8080/v1/ledger/versions/bdd-version/instances/bdd-instance" with body: + """ + {} + """ + Then the HTTP status should be 404 + + When I send a "POST" request to "http://events.localhost:8080/v2/ledger/versions/bdd-version/deployments/bdd-deployment/instances/bdd-instance" with body: + """ + {} + """ + Then the HTTP status should be 404 + + Scenario: A read key can query the ledger and bad queries are rejected + When I successfully run command: + """ + /bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z' + """ + And I export command output to environment variable "BDD_EL_KEY_EXPIRES" + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "POST" request to "http://api-keys.localhost:8080/v1/keys" with body: + """ + { + "description": "bdd-event-ledger-read", + "expires_at": "${BDD_EL_KEY_EXPIRES}", + "authorizations": { + "policies": [ + { + "aud": "nvidia-event-ledger-ncp-service-id-ckozoh6f", + "auds": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"], + "product": "nv-cloud-functions", + "resources": [{"id": "*", "type": "account-functions"}], + "scopes": ["fnds:getEvents", "fnds:getStats"] + } + ] + }, + "audience_service_ids": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"] + } + """ + Then the HTTP status should be 200 + When I export the HTTP response JSON field "value" to environment variable "BDD_EL_READ_KEY" + And I export the HTTP response JSON field "id" to environment variable "BDD_EL_READ_KEY_ID" + # Delete the key when the scenario ends, even if it fails. + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + And I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" at scenario teardown expecting status 204 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/stats" + Then the HTTP status should be 200 + And the HTTP response JSON field "namespace" should equal "bdd-no-such-namespace" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/events" + Then the HTTP status should be 200 + + # Malformed queries must get a 400 before reaching Cassandra: a half-supplied + # attribute filter, a disallowed character in a context id, and an unknown stats view. + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/events?attribute_key=only-a-key" + Then the HTTP status should be 400 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/events?instance_id=bad!id" + Then the HTTP status should be 400 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-no-such-namespace/stats?view=bogus" + Then the HTTP status should be 400 + + + # Write access needs an OpenBao JWT with the FnDS write scope, so a + # read-scoped api key must be rejected on the write route. + Scenario: A read-scoped key cannot write events + When I successfully run command: + """ + /bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z' + """ + And I export command output to environment variable "BDD_EL_KEY_EXPIRES" + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "POST" request to "http://api-keys.localhost:8080/v1/keys" with body: + """ + { + "description": "bdd-event-ledger-read-only-write", + "expires_at": "${BDD_EL_KEY_EXPIRES}", + "authorizations": { + "policies": [ + { + "aud": "nvidia-event-ledger-ncp-service-id-ckozoh6f", + "auds": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"], + "product": "nv-cloud-functions", + "resources": [{"id": "*", "type": "account-functions"}], + "scopes": ["fnds:getEvents", "fnds:getStats"] + } + ] + }, + "audience_service_ids": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"] + } + """ + Then the HTTP status should be 200 + When I export the HTTP response JSON field "value" to environment variable "BDD_EL_READ_KEY" + And I export the HTTP response JSON field "id" to environment variable "BDD_EL_READ_KEY_ID" + # Delete the key when the scenario ends, even if it fails. + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + And I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" at scenario teardown expecting status 204 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + And the HTTP request header "Content-Type" is "application/cloudevents+json" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/cloudevents" with body: + """ + {"specversion":"1.0","id":"bdd-write-attempt-1","type":"BddWriteAttempt","source":"bdd","namespace":"bdd-write-attempt"} + """ + Then the HTTP status should be 401 or 403 + + # The other write route is registered separately, so check it too. The scope + # check runs before the body is read, so no payload is needed. + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + And the HTTP request header "Content-Type" is "application/x-protobuf" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/k8s-events" + Then the HTTP status should be 403 + + + # The positive control for the scope checks: a key with only the write scope + # may write, which shows the rejections above are specific and not an outage, + # and it must not be able to read. The event is fixed, so reruns update one row. + Scenario: A write-scoped key can write events but cannot read them + When I successfully run command: + """ + /bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z' + """ + And I export command output to environment variable "BDD_EL_KEY_EXPIRES" + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "POST" request to "http://api-keys.localhost:8080/v1/keys" with body: + """ + { + "description": "bdd-event-ledger-write-only", + "expires_at": "${BDD_EL_KEY_EXPIRES}", + "authorizations": { + "policies": [ + { + "aud": "nvidia-event-ledger-ncp-service-id-ckozoh6f", + "auds": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"], + "product": "nv-cloud-functions", + "resources": [{"id": "*", "type": "account-functions"}], + "scopes": ["fnds:createEvent"] + } + ] + }, + "audience_service_ids": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"] + } + """ + Then the HTTP status should be 200 + When I export the HTTP response JSON field "value" to environment variable "BDD_EL_WRITE_KEY" + And I export the HTTP response JSON field "id" to environment variable "BDD_EL_WRITE_KEY_ID" + # Delete the key when the scenario ends, even if it fails. + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + And I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_WRITE_KEY_ID}" at scenario teardown expecting status 204 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" + And the HTTP request header "Content-Type" is "application/cloudevents+json" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/cloudevents" with body: + """ + {"specversion":"1.0","id":"bdd-write-scope-1","type":"BddWriteScope","source":"bdd","namespace":"bdd-write-scope"} + """ + Then the HTTP status should be 200 + And the HTTP response JSON field "status" should equal "accepted" + And the HTTP response JSON field "success_count" should be at least 1 + + # An empty OTLP body is rejected by the handler, so a 400 here shows the key + # passed the scope check on this route without writing anything. + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" + And the HTTP request header "Content-Type" is "application/x-protobuf" + When I send a "POST" request to "http://events.localhost:8080/v3/ledger/k8s-events" + Then the HTTP status should be 400 + And the HTTP response body should contain "No events found in request" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-write-scope/events" + Then the HTTP status should be 403 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_WRITE_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/bdd-write-scope/stats" + Then the HTTP status should be 403 + + + Rule: Helmfile installs NVCA on the same local cluster after registration + + Background: + Given these environment variables are set: + | name | + | NVCF_CLI | + | REPO_ROOT | + # This rule depends on the earlier control-plane install scenario in the + # same feature run. It is only needed for the function deployment scenario. + + @nvca-registration @compute-plane + Scenario: Operator registers the local cluster and installs the NVCA operator + When I successfully run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml self-hosted --control-plane-stack deploy/stacks/self-managed --env local-bdd-event-ledger control-plane profile export --cluster-name ncp-local + """ + Then file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should exist + + When I successfully run command: + """ + make -C deploy/stacks/nvcf-compute-plane register-cluster CLUSTER_NAME=ncp-local CONTROL_PLANE_PROFILE=${REPO_ROOT}/deploy/stacks/self-managed/out/control-plane-profile.yaml COMPUTE_KUBE_CONTEXT=k3d-ncp-local NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml + """ + Then file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-register-values.yaml" should exist + And yaml file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-register-values.yaml" should have non-empty keys: + | key | + | clusterID | + | clusterGroupID | + + When I successfully run command: + """ + make -C deploy/stacks/nvcf-compute-plane install CLUSTER_NAME=ncp-local HELMFILE_ENV=local-bdd-event-ledger NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml + """ + + Then these Helm releases should be deployed using context "k3d-ncp-local": + | name | namespace | + | nvca-operator | nvca-operator | + + Then deployment "nvca-operator" in namespace "nvca-operator" using context "k3d-ncp-local" should complete rollout within "10m" + + Then NVCFBackend "ncp-local" in namespace "nvca-operator" using context "k3d-ncp-local" should report agent status "healthy" within "10m" + + Rule: A function deployment produces SIS events + + Background: + Given these environment variables are set: + | name | + | NGC_API_KEY | + | SAMPLE_NGC_ORG | + | SAMPLE_NGC_TEAM | + | REPO_ROOT | + And I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml" + + # This scenario depends on the earlier control-plane install and NVCA + # registration scenarios in the same feature run. + @compute-plane @function-lifecycle + Scenario: A function deployment produces SIS events that can be read back + When I successfully run command: + """ + /bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z' + """ + And I export command output to environment variable "BDD_EL_KEY_EXPIRES" + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Content-Type" is "application/json" + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + When I send a "POST" request to "http://api-keys.localhost:8080/v1/keys" with body: + """ + { + "description": "bdd-event-ledger-lifecycle", + "expires_at": "${BDD_EL_KEY_EXPIRES}", + "authorizations": { + "policies": [ + { + "aud": "nvidia-event-ledger-ncp-service-id-ckozoh6f", + "auds": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"], + "product": "nv-cloud-functions", + "resources": [{"id": "*", "type": "account-functions"}], + "scopes": ["fnds:getEvents", "fnds:getStats"] + } + ] + }, + "audience_service_ids": ["nvidia-event-ledger-ncp-service-id-ckozoh6f"] + } + """ + Then the HTTP status should be 200 + When I export the HTTP response JSON field "value" to environment variable "BDD_EL_READ_KEY" + And I export the HTTP response JSON field "id" to environment variable "BDD_EL_READ_KEY_ID" + # Delete the key when the scenario ends, even if it fails. + Given the HTTP request header "Authorization" is the NVCF CLI admin token with prefix "Bearer " + And the HTTP request header "Key-Issuer-Service" is "nvcf-api" + And the HTTP request header "Key-Issuer-Id" is "nvidia-event-ledger-ncp-service-id-ckozoh6f" + And the HTTP request header "Key-Owner-Id" is "svc@nvcf-api.local" + And I send a "DELETE" request to "http://api-keys.localhost:8080/v1/keys/${BDD_EL_READ_KEY_ID}" at scenario teardown expecting status 204 + + When I successfully create function "bdd-event-ledger" from image "nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/load_tester_supreme:0.0.8" with CLI options: + | option | value | + | --inference-url | /echo | + | --inference-port | 8000 | + | --health-uri | /health | + | --health-port | 8000 | + | --health-timeout | PT30S | + And I successfully deploy the function selected by NVCF CLI with options: + | option | value | + | --gpu | H100 | + | --instance-type | NCP.GPU.H100_1x | + | --backend | ncp-local | + | --regions | us-west-1 | + | --min-instances | 1 | + | --max-instances | 1 | + | --timeout | 900 | + # Undeploy when the scenario ends if it has not already, even if it fails. + And the function selected by NVCF CLI is undeployed at scenario teardown + And I export the function selected by NVCF CLI to environment variables "BDD_EL_FUNCTION_ID" and "BDD_EL_VERSION_ID" + + # Events are stored per context, and stats list every context in the + # namespace with its latest event. The NVCA collector, when enabled, adds + # contexts of its own that sort before SIS's, so find the SIS context with + # an event filter, then read its events. + + # SIS writes InstanceReady once the instance is running. + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I poll "GET" "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats" for up to 300 seconds until the HTTP response body contains "InstanceReady" + Then the HTTP status should be 200 + And the HTTP response JSON field "namespace" should equal "${BDD_EL_VERSION_ID}" + And the HTTP response JSON field "summary.by_event.InstanceReady" should be at least 1 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats?eventFilter=InstanceReady" + Then the HTTP status should be 200 + When I export the HTTP response JSON field "contexts.0.context" to environment variable "BDD_EL_CONTEXT" + And I successfully run command: + """ + /bin/bash -c 'echo "${BDD_EL_CONTEXT}" | tr "," "&"' + """ + And I export command output to environment variable "BDD_EL_CONTEXT_QUERY" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/events?${BDD_EL_CONTEXT_QUERY}" + Then the HTTP status should be 200 + And the HTTP response body should contain "InstanceReady" + And the HTTP response body should contain "nvidia-spot" + And the HTTP response body should contain "${BDD_EL_FUNCTION_ID}" + And the HTTP response body should not contain "InstanceDestroyed" + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats?eventFilter=NoSuchEvent" + Then the HTTP status should be 200 + And the HTTP response body should not contain "InstanceReady" + + # Undeploy removes the instance and SIS then reports it destroyed. Stats keep + # only the latest event per context, while events keep the whole history. + When I successfully undeploy the function selected by NVCF CLI + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I poll "GET" "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/stats" for up to 180 seconds until the HTTP response body contains "InstanceDestroyed" + Then the HTTP status should be 200 + And the HTTP response JSON field "summary.by_event.InstanceDestroyed" should be at least 1 + + Given the HTTP request header "Authorization" is "Bearer ${BDD_EL_READ_KEY}" + When I send a "GET" request to "http://events.localhost:8080/v3/ledger/namespace/${BDD_EL_VERSION_ID}/events?${BDD_EL_CONTEXT_QUERY}" + Then the HTTP status should be 200 + And the HTTP response body should contain "InstanceReady" + And the HTTP response body should contain "InstanceDestroyed" diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 9e2d61ca78..d67301006a 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -21,7 +21,10 @@ package bdd_tmp import ( "context" + "encoding/json" "io" + "net/http" + "net/url" "os" "path/filepath" "strings" @@ -2663,6 +2666,28 @@ func TestObservabilityDisabled(t *testing.T) { runLiveFeature(t, "observability-disabled.feature") } +// TestEventLedger is the live entry point for the event-ledger feature. It +// brings up a single-cluster ncp-local cluster, installs the stack with the +// event ledger enabled, registers the cluster and installs NVCA, then checks +// the ledger. Skipped under -short. +func TestEventLedger(t *testing.T) { + if testing.Short() { + t.Skip("live run skipped under -short") + } + runLiveFeature(t, "event-ledger.feature") +} + +// TestEventLedgerControlPlane runs the event-ledger feature without the +// compute-plane scenarios (NVCA registration and the function deployment), so +// it needs no GPU capacity or image pull access. It still installs the stack. +// Skipped under -short. +func TestEventLedgerControlPlane(t *testing.T) { + if testing.Short() { + t.Skip("live run skipped under -short") + } + runLiveFeatureTags(t, "event-ledger.feature", "~@compute-plane") +} + // TestMultiClusterHelmfile is the live entry point for the // multi-cluster Helmfile feature: control-plane install on // k3d-ncp-local-cp followed by register + NVCA install on the @@ -2888,3 +2913,324 @@ func mustResolveFeaturePath(t *testing.T, name string) string { t.Helper() return filepath.Join("features", name) } + +// eventLedgerHTTPFake answers the event-ledger feature's HTTP calls the way +// the real services are expected to, and records requests for assertions. +type eventLedgerHTTPFake struct { + destroyed *bool + requests []string +} + +func (f *eventLedgerHTTPFake) Do(_ context.Context, method, rawURL string, headers map[string]string, body string) (harness.HTTPResponse, error) { + f.requests = append(f.requests, method+" "+rawURL) + hostAndPath := strings.TrimPrefix(rawURL, "http://") + host, rest, _ := strings.Cut(hostAndPath, "/") + path, query, _ := strings.Cut("/"+rest, "?") + auth := headers["Authorization"] + admin := auth == "Bearer admin-jwt" + reader := auth == "Bearer nvapi-test" + writer := auth == "Bearer nvapi-write" + + if strings.HasPrefix(host, "api-keys.") { + switch { + case !admin || headers["Key-Issuer-Id"] == "": + return harness.HTTPResponse{Status: 401}, nil + case method == "POST" && strings.Contains(body, "fnds:createEvent"): + return harness.HTTPResponse{Status: 200, Body: `{"id":"key-w","value":"nvapi-write"}`}, nil + case method == "POST": + return harness.HTTPResponse{Status: 200, Body: `{"id":"key-1","value":"nvapi-test"}`}, nil + default: + return harness.HTTPResponse{Status: 204}, nil + } + } + switch { + case path == "/health": + return harness.HTTPResponse{Status: 200}, nil + case strings.HasPrefix(path, "/v1/"), strings.HasPrefix(path, "/v2/"): + return harness.HTTPResponse{Status: 404}, nil + case auth == "", strings.HasPrefix(auth, "Bearer eyJ"): + return harness.HTTPResponse{Status: 401}, nil + case writer && method == "POST" && path == "/v3/ledger/k8s-events": + return harness.HTTPResponse{Status: 400, Body: `{"title":"Bad Request","status":400,"detail":"No events found in request"}`}, nil + case writer && method == "POST": + return harness.HTTPResponse{Status: 200, Body: `{"status":"accepted","success_count":1,"failure_count":0,"total_processed":1}`}, nil + case !reader: + return harness.HTTPResponse{Status: 403}, nil + case method == "POST": + return harness.HTTPResponse{Status: 403}, nil + } + namespace := strings.Split(path, "/")[4] + params, _ := url.ParseQuery(query) + switch { + case params.Has("attribute_key") || strings.HasPrefix(params.Get("instance_id"), "bad") || params.Has("view"): + return harness.HTTPResponse{Status: 400}, nil + case strings.HasSuffix(path, "/stats"): + return f.stats(namespace, params.Get("eventFilter")), nil + default: + return f.events(namespace, params), nil + } +} + +// Contexts the fake keeps for the function version "version-1": the SIS one +// and an NVCA one that sorts before it, as the ledger lists them when the NVCA +// collector is enabled. The scenario must still pick the SIS context. +const ( + fakeNVCAContext = "cluster_id=c1,instance_id=i1,resource_id=r1" + fakeSISContext = "deployment_id=d1,gpu_specification_id=g1,instance_id=i1" +) + +type fakeEvent struct{ name, source string } + +func (f *eventLedgerHTTPFake) contextEvents() map[string][]fakeEvent { + sis := []fakeEvent{{"InstanceReady", "nvidia-spot"}} + if *f.destroyed { + sis = append(sis, fakeEvent{"InstanceDestroyed", "nvidia-spot"}) + } + return map[string][]fakeEvent{ + fakeNVCAContext: {{"ICMSRequest.InstanceStatusUpdate.running", "nvca"}}, + fakeSISContext: sis, + } +} + +func (f *eventLedgerHTTPFake) stats(namespace, eventFilter string) harness.HTTPResponse { + type row struct { + Context string `json:"context"` + EventName string `json:"event_name"` + } + contexts := []row{} + byEvent := map[string]int{} + if namespace == "version-1" { + events := f.contextEvents() + for _, ctx := range []string{fakeNVCAContext, fakeSISContext} { + latest := events[ctx][len(events[ctx])-1].name + if eventFilter != "" && !strings.Contains(","+eventFilter+",", ","+latest+",") { + continue + } + contexts = append(contexts, row{ctx, latest}) + byEvent[latest]++ + } + } + body, _ := json.Marshal(map[string]any{ + "namespace": namespace, + "summary": map[string]any{"by_event": byEvent}, + "contexts": contexts, + }) + return harness.HTTPResponse{Status: 200, Body: string(body)} +} + +func (f *eventLedgerHTTPFake) events(namespace string, params url.Values) harness.HTTPResponse { + var parts []string + for _, key := range []string{"cluster_id", "deployment_id", "gpu_specification_id", "instance_id", "resource_id"} { + if value := params.Get(key); value != "" { + parts = append(parts, key+"="+value) + } + } + eventContext := strings.Join(parts, ",") + type event struct { + Name string `json:"event_name"` + Source string `json:"source"` + Details map[string]any `json:"details"` + } + events := []event{} + if namespace == "version-1" { + for _, e := range f.contextEvents()[eventContext] { + events = append(events, event{e.name, e.source, map[string]any{"attributes": map[string]string{"functionid": "function-1", "function_id": "function-1"}}}) + } + } + body, _ := json.Marshal(map[string]any{"namespace": namespace, "context": eventContext, "events": events}) + return harness.HTTPResponse{Status: 200, Body: string(body)} +} + +func eventLedgerHelmListJSON() string { + return `[ +{"name":"nats","namespace":"nats-system","status":"deployed"}, +{"name":"cert-manager","namespace":"cert-manager","status":"deployed"}, +{"name":"openbao-server","namespace":"vault-system","status":"deployed"}, +{"name":"cassandra","namespace":"cassandra-system","status":"deployed"}, +{"name":"api-keys","namespace":"api-keys","status":"deployed"}, +{"name":"sis","namespace":"sis","status":"deployed"}, +{"name":"api","namespace":"nvcf","status":"deployed"}, +{"name":"event-ledger","namespace":"nvcf","status":"deployed"}, +{"name":"ingress","namespace":"envoy-gateway-system","status":"deployed"}, +{"name":"nvca-operator","namespace":"nvca-operator","status":"deployed"} +]` +} + +// recordingHTTP is an HTTPClient that records requests and answers every one +// with a fixed status. +type recordingHTTP struct { + status int + requests []string + headers []map[string]string +} + +func (r *recordingHTTP) Do(_ context.Context, method, url string, headers map[string]string, _ string) (harness.HTTPResponse, error) { + r.requests = append(r.requests, method+" "+url) + r.headers = append(r.headers, headers) + return harness.HTTPResponse{Status: r.status}, nil +} + +func runTeardownFeature(t *testing.T, body string, httpClient harness.HTTPClient) int { + t.Helper() + feature := filepath.Join(t.TempDir(), "teardown.feature") + if err := os.WriteFile(feature, []byte(body), 0o644); err != nil { + t.Fatalf("write feature: %v", err) + } + suite := newWiringSuite(t, newFakeRunner(nil)) + suite.HTTP = httpClient + sc := steps.NewScenarioContext(suite) + return godog.TestSuite{ + Name: "teardown-wiring", + ScenarioInitializer: func(ctx *godog.ScenarioContext) { + steps.RegisterAll(ctx, sc) + }, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{feature}, + Strict: true, + StopOnFailure: true, + Output: io.Discard, + }, + }.Run() +} + +// TestScenarioTeardownRunsAfterAFailedStep guards the live runner stopping at +// the first failed step: cleanup written as a later step would be skipped, a +// teardown must not be. +func TestScenarioTeardownRunsAfterAFailedStep(t *testing.T) { + recorder := &recordingHTTP{status: http.StatusNoContent} + status := runTeardownFeature(t, `Feature: teardown + Scenario: cleanup survives a failed step + Given the HTTP request header "Key-Owner-Id" is "owner" + And I send a "DELETE" request to "http://api-keys/v1/keys/key-1" at scenario teardown expecting status 204 + Then the HTTP status should be 200 +`, recorder) + if status == 0 { + t.Fatal("the scenario should fail on its assertion") + } + if len(recorder.requests) != 1 || recorder.requests[0] != "DELETE http://api-keys/v1/keys/key-1" { + t.Fatalf("teardown requests = %v", recorder.requests) + } + if recorder.headers[0]["Key-Owner-Id"] != "owner" { + t.Fatalf("teardown headers = %v", recorder.headers[0]) + } +} + +func TestScenarioTeardownFailureFailsAnOtherwisePassingScenario(t *testing.T) { + recorder := &recordingHTTP{status: http.StatusInternalServerError} + status := runTeardownFeature(t, `Feature: teardown + Scenario: cleanup that fails is reported + When I send a "GET" request to "http://example/ok" + And I send a "DELETE" request to "http://api-keys/v1/keys/key-1" at scenario teardown expecting status 204 +`, recorder) + if status == 0 { + t.Fatal("a failed teardown should fail the scenario") + } +} + +func TestEventLedgerFeatureFileWiresToSteps(t *testing.T) { + const selectedFunctionStatusCommand = `/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml status --json` + home := t.TempDir() + t.Setenv("HOME", home) + if err := os.WriteFile(filepath.Join(home, ".nvcf-cli.nvcf-cli-local.state"), []byte(`{"token":"admin-jwt"}`), 0o600); err != nil { + t.Fatalf("seed cli state: %v", err) + } + t.Setenv("NGC_API_KEY", "test-key") + t.Setenv("SAMPLE_NGC_ORG", "test-org") + t.Setenv("SAMPLE_NGC_TEAM", "test-team") + t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") + t.Setenv("REPO_ROOT", "/repo-root-placeholder") + destroyed := false + runner := newFakeRunner(map[string]harness.Result{ + // Conflict precheck: the multi-cluster control plane must be absent. + "k3d cluster get ncp-local-cp": {ExitCode: 1}, + "helm list --all-namespaces --kube-context k3d-ncp-local -o json": { + ExitCode: 0, + Stdout: eventLedgerHelmListJSON(), + }, + selectedFunctionStatusCommand: { + ExitCode: 0, + Stdout: `{"currentFunction":{"hasFunction":true,"functionId":"function-1","versionId":"version-1"}}`, + }, + `/bin/bash -c 'date -u -d "+1 day" +%Y-%m-%dT%H:%M:%S.000Z'`: {ExitCode: 0, Stdout: "2030-01-01T00:00:00.000Z\n"}, + `/bin/bash -c 'echo "` + fakeSISContext + `" | tr "," "&"'`: { + ExitCode: 0, + Stdout: "deployment_id=d1&gpu_specification_id=g1&instance_id=i1\n", + }, + "kubectl --context k3d-ncp-local --namespace nvcf get service event-ledger -o jsonpath={.spec.ports[*].name}": { + ExitCode: 0, + Stdout: "api-port metrics", + }, + "kubectl --context k3d-ncp-local --namespace sis get configmaps -o yaml": { + ExitCode: 0, + Stdout: " ICMS_FNDS_MESSAGES_ENABLED: \"true\"\n ICMS_FNDS_MESSAGES_V3_ENABLED: \"true\"\n", + }, + }) + runner.onRun = func(command string) error { + if strings.Contains(command, " function delete --deployment-only") { + destroyed = true + } + return nil + } + suite := newWiringSuite(t, runner) + httpFake := &eventLedgerHTTPFake{destroyed: &destroyed} + suite.HTTP = httpFake + seedHelmfileLocalBDDFixture(t, suite.Config.RepoRoot) + seedComputePlaneLocalBDDFixture(t, suite.Config.RepoRoot) + seedStackSecretsTemplate(t, suite.Config.RepoRoot) + writeProfileHandoffArtifact(t, suite.Config.RepoRoot) + writeHelmfileRegisterValues(t, suite.Config.RepoRoot) + + sc := steps.NewScenarioContext(suite) + featurePath := mustResolveFeaturePath(t, "event-ledger.feature") + var out strings.Builder + status := godog.TestSuite{ + Name: "event-ledger-wiring", + ScenarioInitializer: func(ctx *godog.ScenarioContext) { + steps.RegisterAll(ctx, sc) + }, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{featurePath}, + Strict: true, + Output: &out, + }, + }.Run() + if status != 0 { + t.Fatalf("godog suite status = %d\n%s", status, out.String()) + } + runs := suite.Runner.(*fakeRunner).runs + for _, command := range []string{ + "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-event-ledger", + "make -C deploy/stacks/nvcf-compute-plane register-cluster CLUSTER_NAME=ncp-local", + "make -C deploy/stacks/nvcf-compute-plane install CLUSTER_NAME=ncp-local HELMFILE_ENV=local-bdd-event-ledger", + "nvcf-cli-local.yaml init", + } { + if !commandRanThatContains(runs, command) { + t.Fatalf("install flow command was never invoked: %s", command) + } + } + if !commandRanThatContains(runs, " function deploy create ") { + t.Fatal("function deploy command was never invoked") + } + if !commandRanThatContains(runs, " function delete --deployment-only") { + t.Fatal("function undeploy command was never invoked") + } + for _, secret := range []string{"admin-jwt", "nvapi-test", "nvapi-write"} { + if commandRanThatContains(runs, secret) { + t.Fatalf("secret %q leaked into a command line", secret) + } + } + readKeyDeletes, writeKeyDeletes := 0, 0 + for _, request := range httpFake.requests { + switch request { + case "DELETE http://api-keys.localhost:8080/v1/keys/key-1": + readKeyDeletes++ + case "DELETE http://api-keys.localhost:8080/v1/keys/key-w": + writeKeyDeletes++ + } + } + if readKeyDeletes != 3 || writeKeyDeletes != 1 { + t.Fatalf("keys deleted: read = %d (want 3), write = %d (want 1)", readKeyDeletes, writeKeyDeletes) + } +} diff --git a/tests/bdd/harness/http.go b/tests/bdd/harness/http.go new file mode 100644 index 0000000000..449d67b379 --- /dev/null +++ b/tests/bdd/harness/http.go @@ -0,0 +1,82 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package harness + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +const httpTimeout = 30 * time.Second + +// HTTPResponse is the status code and body of one HTTP exchange. +type HTTPResponse struct { + Status int + Body string +} + +// HTTPClient sends one HTTP request. Header values may carry credentials, +// so implementations must not log them. +type HTTPClient interface { + Do(ctx context.Context, method, url string, headers map[string]string, body string) (HTTPResponse, error) +} + +type netHTTPClient struct { + client *http.Client +} + +// NewHTTPClient returns an HTTPClient backed by net/http. Redirects are not +// followed so a test can assert on a 301 directly. +func NewHTTPClient() HTTPClient { + return &netHTTPClient{client: &http.Client{ + Timeout: httpTimeout, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }} +} + +// Do sends the request and returns the status and full body. The error text +// never includes header values. +func (c *netHTTPClient) Do(ctx context.Context, method, url string, headers map[string]string, body string) (HTTPResponse, error) { + var reader io.Reader + if body != "" { + reader = strings.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, url, reader) + if err != nil { + return HTTPResponse{}, fmt.Errorf("build %s request to %s: %w", method, url, err) + } + for name, value := range headers { + req.Header.Set(name, value) + } + resp, err := c.client.Do(req) + if err != nil { + return HTTPResponse{}, fmt.Errorf("%s %s: %w", method, url, err) + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + return HTTPResponse{}, fmt.Errorf("read %s %s response: %w", method, url, err) + } + return HTTPResponse{Status: resp.StatusCode, Body: string(data)}, nil +} diff --git a/tests/bdd/harness/http_test.go b/tests/bdd/harness/http_test.go new file mode 100644 index 0000000000..1ca65ce2b0 --- /dev/null +++ b/tests/bdd/harness/http_test.go @@ -0,0 +1,75 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package harness + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestNetHTTPClientSendsHeadersAndBody(t *testing.T) { + var gotAuth, gotBody, gotMethod string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + gotMethod = r.Method + data, _ := io.ReadAll(r.Body) + gotBody = string(data) + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + defer server.Close() + + resp, err := NewHTTPClient().Do(context.Background(), "POST", server.URL, map[string]string{"Authorization": "Bearer t"}, `{"a":1}`) + if err != nil { + t.Fatal(err) + } + if resp.Status != http.StatusCreated || resp.Body != `{"ok":true}` { + t.Fatalf("resp = %+v", resp) + } + if gotAuth != "Bearer t" || gotBody != `{"a":1}` || gotMethod != "POST" { + t.Fatalf("server saw auth=%q body=%q method=%q", gotAuth, gotBody, gotMethod) + } +} + +func TestNetHTTPClientDoesNotFollowRedirects(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "/elsewhere", http.StatusMovedPermanently) + })) + defer server.Close() + resp, err := NewHTTPClient().Do(context.Background(), "GET", server.URL, nil, "") + if err != nil { + t.Fatal(err) + } + if resp.Status != http.StatusMovedPermanently { + t.Fatalf("status = %d, want 301", resp.Status) + } +} + +func TestNetHTTPClientErrorOmitsHeaderValues(t *testing.T) { + _, err := NewHTTPClient().Do(context.Background(), "GET", "http://127.0.0.1:1", map[string]string{"Authorization": "Bearer secret-token"}, "") + if err == nil { + t.Fatal("expected connection error") + } + if strings.Contains(err.Error(), "secret-token") { + t.Fatalf("error leaks header value: %v", err) + } +} diff --git a/tests/bdd/harness/suite.go b/tests/bdd/harness/suite.go index 9094c685eb..ff967ae351 100644 --- a/tests/bdd/harness/suite.go +++ b/tests/bdd/harness/suite.go @@ -40,6 +40,7 @@ type Suite struct { Ledger *Ledger EnvLedger *EnvLedger Cache *CommandCache + HTTP HTTPClient signalMu sync.Mutex signalContext context.Context diff --git a/tests/bdd/steps/context.go b/tests/bdd/steps/context.go index a5683a7bd6..d756a7d3ec 100644 --- a/tests/bdd/steps/context.go +++ b/tests/bdd/steps/context.go @@ -54,6 +54,19 @@ type ScenarioContext struct { LastCommand string NVCFCLIConfig string Manifests map[string]string + HTTPHeaders map[string]string + LastHTTP *harness.HTTPResponse + + // FunctionUndeployed is set once the scenario has undeployed the selected + // function, so the teardown does not undeploy it again. + FunctionUndeployed bool + teardown []teardownAction +} + +// teardownAction is cleanup queued for the end of the scenario. +type teardownAction struct { + description string + run func(context.Context) error } // NewScenarioContext wraps suite in a fresh per-scenario state. The @@ -72,8 +85,17 @@ func RegisterAll(ctx *godog.ScenarioContext, sc *ScenarioContext) { sc.LastCommand = "" sc.NVCFCLIConfig = "" sc.Manifests = nil + sc.HTTPHeaders = nil + sc.LastHTTP = nil + sc.FunctionUndeployed = false + sc.teardown = nil return c, nil }) + // Godog runs After hooks for failed scenarios too, which is what lets + // cleanup survive the live runner stopping at the first failed step. + ctx.After(func(c context.Context, _ *godog.Scenario, _ error) (context.Context, error) { + return c, sc.runTeardown(c) + }) // Godog's default pretty formatter buffers the scenario block until // every step finishes, which makes hangs invisible during a live // run. Echo each step's text to stderr the moment it starts so the @@ -101,6 +123,8 @@ func RegisterAll(ctx *godog.ScenarioContext, sc *ScenarioContext) { registerFileSteps(ctx, sc) registerManifestSteps(ctx, sc) registerCommandSteps(ctx, sc) + registerHTTPSteps(ctx, sc) + registerTeardownSteps(ctx, sc) registerNVCFCLISteps(ctx, sc) registerRegistrationSteps(ctx, sc) registerAssertionSteps(ctx, sc) diff --git a/tests/bdd/steps/http_steps.go b/tests/bdd/steps/http_steps.go new file mode 100644 index 0000000000..c135ad15dd --- /dev/null +++ b/tests/bdd/steps/http_steps.go @@ -0,0 +1,264 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package steps + +import ( + "context" + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/cucumber/godog" + + "nvcf-bdd/dsl" + "nvcf-bdd/harness" +) + +const maxHTTPFailureBody = 300 + +// httpPollInterval is the pause between polls; tests shorten it. +var httpPollInterval = 5 * time.Second + +// registerHTTPSteps hooks the HTTP request and response assertion steps. +// Request headers are declared with their own steps and apply to the next +// request only, so a header value never appears in the send step text. +func registerHTTPSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { + ctx.Step(`^the HTTP request header "([^"]*)" is "([^"]*)"$`, sc.theHTTPRequestHeaderIs) + ctx.Step(`^the HTTP request header "([^"]*)" is the NVCF CLI admin token with prefix "([^"]*)"$`, sc.theHTTPRequestHeaderIsAdminToken) + ctx.Step(`^I send an? "([A-Z]+)" request to "([^"]*)"$`, sc.iSendHTTPRequest) + ctx.Step(`^I send an? "([A-Z]+)" request to "([^"]*)" with body:$`, sc.iSendHTTPRequestWithBody) + ctx.Step(`^I poll "([A-Z]+)" "([^"]*)" for up to (\d+) seconds until the HTTP response body contains "([^"]*)"$`, sc.iPollHTTPUntilBodyContains) + ctx.Step(`^the HTTP status should be (\d+)$`, sc.theHTTPStatusShouldBe) + ctx.Step(`^the HTTP status should be (\d+) or (\d+)$`, sc.theHTTPStatusShouldBeOneOf) + ctx.Step(`^the HTTP response body should contain "([^"]*)"$`, sc.theHTTPBodyShouldContain) + ctx.Step(`^the HTTP response body should not contain "([^"]*)"$`, sc.theHTTPBodyShouldNotContain) + ctx.Step(`^the HTTP response JSON field "([^"]*)" should equal "([^"]*)"$`, sc.theHTTPJSONFieldShouldEqual) + ctx.Step(`^the HTTP response JSON field "([^"]*)" should be at least (\d+)$`, sc.theHTTPJSONFieldShouldBeAtLeast) + ctx.Step(`^I export the HTTP response JSON field "([^"]*)" to environment variable "([^"]*)"$`, sc.iExportHTTPJSONField) +} + +func (sc *ScenarioContext) httpClient() harness.HTTPClient { + if sc.Suite.HTTP == nil { + sc.Suite.HTTP = harness.NewHTTPClient() + } + return sc.Suite.HTTP +} + +func (sc *ScenarioContext) theHTTPRequestHeaderIs(name, value string) error { + if strings.TrimSpace(name) == "" { + return fmt.Errorf("http header name must be non-empty") + } + if sc.HTTPHeaders == nil { + sc.HTTPHeaders = map[string]string{} + } + sc.HTTPHeaders[name] = dsl.Interpolate(value) + return nil +} + +// theHTTPRequestHeaderIsAdminToken reads the admin token from the state file +// of the selected NVCF CLI config so the JWT never enters argv, env or logs. +func (sc *ScenarioContext) theHTTPRequestHeaderIsAdminToken(name, prefix string) error { + if sc.NVCFCLIConfig == "" { + return fmt.Errorf("no NVCF CLI config selected; use 'I use NVCF CLI config' first") + } + home, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("resolve home dir: %w", err) + } + token, err := dsl.CLIStateToken(dsl.CLIStatePath(home, sc.NVCFCLIConfig)) + if err != nil { + return err + } + if sc.HTTPHeaders == nil { + sc.HTTPHeaders = map[string]string{} + } + sc.HTTPHeaders[name] = prefix + token + return nil +} + +func (sc *ScenarioContext) iSendHTTPRequest(ctx context.Context, method, url string) error { + return sc.sendHTTP(ctx, method, url, "") +} + +func (sc *ScenarioContext) iSendHTTPRequestWithBody(ctx context.Context, method, url string, doc *godog.DocString) error { + return sc.sendHTTP(ctx, method, url, doc.Content) +} + +// sendHTTP interpolates the URL and body, sends one request with the pending +// headers, and records the response. A non-2xx status is not a step failure; +// the status assertion decides. +func (sc *ScenarioContext) sendHTTP(ctx context.Context, method, url, body string) error { + headers := sc.HTTPHeaders + sc.HTTPHeaders = nil + resp, err := sc.httpClient().Do(ctx, method, dsl.Interpolate(url), headers, dsl.Interpolate(body)) + if err != nil { + sc.LastHTTP = nil + return err + } + sc.LastHTTP = &resp + return nil +} + +func (sc *ScenarioContext) iPollHTTPUntilBodyContains(ctx context.Context, method, url string, seconds int, text string) error { + want := dsl.Interpolate(text) + if strings.TrimSpace(want) == "" { + return fmt.Errorf("poll text must be non-empty") + } + headers := sc.HTTPHeaders + sc.HTTPHeaders = nil + deadline := time.Now().Add(time.Duration(seconds) * time.Second) + for { + resp, err := sc.httpClient().Do(ctx, method, dsl.Interpolate(url), headers, "") + if err == nil { + sc.LastHTTP = &resp + if strings.Contains(resp.Body, want) { + return nil + } + } + if time.Now().Add(httpPollInterval).After(deadline) { + if err != nil { + return fmt.Errorf("poll timed out after %d seconds: %w", seconds, err) + } + return fmt.Errorf("poll timed out after %d seconds: last status %d, body missing %q: %s", + seconds, sc.LastHTTP.Status, want, truncateBody(sc.LastHTTP.Body)) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(httpPollInterval): + } + } +} + +func (sc *ScenarioContext) requireHTTP() (harness.HTTPResponse, error) { + if sc.LastHTTP == nil { + return harness.HTTPResponse{}, fmt.Errorf("no HTTP response recorded; send a request first") + } + return *sc.LastHTTP, nil +} + +func (sc *ScenarioContext) theHTTPStatusShouldBe(want int) error { + resp, err := sc.requireHTTP() + if err != nil { + return err + } + if resp.Status != want { + return fmt.Errorf("http status = %d, want %d: %s", resp.Status, want, truncateBody(resp.Body)) + } + return nil +} + +func (sc *ScenarioContext) theHTTPStatusShouldBeOneOf(first, second int) error { + resp, err := sc.requireHTTP() + if err != nil { + return err + } + if resp.Status != first && resp.Status != second { + return fmt.Errorf("http status = %d, want %d or %d: %s", resp.Status, first, second, truncateBody(resp.Body)) + } + return nil +} + +func (sc *ScenarioContext) theHTTPBodyShouldContain(text string) error { + resp, err := sc.requireHTTP() + if err != nil { + return err + } + want := dsl.Interpolate(text) + if strings.TrimSpace(want) == "" { + return fmt.Errorf("expected text must be non-empty") + } + if !strings.Contains(resp.Body, want) { + return fmt.Errorf("http response body missing %q: %s", want, truncateBody(resp.Body)) + } + return nil +} + +func (sc *ScenarioContext) theHTTPBodyShouldNotContain(text string) error { + resp, err := sc.requireHTTP() + if err != nil { + return err + } + unwanted := dsl.Interpolate(text) + if strings.TrimSpace(unwanted) == "" { + return fmt.Errorf("unwanted text must be non-empty") + } + if strings.Contains(resp.Body, unwanted) { + return fmt.Errorf("http response body unexpectedly contains %q", unwanted) + } + return nil +} + +func (sc *ScenarioContext) theHTTPJSONFieldShouldEqual(path, want string) error { + resp, err := sc.requireHTTP() + if err != nil { + return err + } + got, err := dsl.JSONField(resp.Body, path) + if err != nil { + return err + } + if expected := dsl.Interpolate(want); got != expected { + return fmt.Errorf("json field %q = %q, want %q", path, got, expected) + } + return nil +} + +func (sc *ScenarioContext) theHTTPJSONFieldShouldBeAtLeast(path string, min int) error { + resp, err := sc.requireHTTP() + if err != nil { + return err + } + return dsl.JSONFieldAtLeast(resp.Body, path, float64(min)) +} + +// iExportHTTPJSONField stores a response field in an env var through the +// EnvLedger, which restores the prior value at suite teardown. The value is +// never echoed. +func (sc *ScenarioContext) iExportHTTPJSONField(path, name string) error { + if name == "" { + return fmt.Errorf("export: env var name must be non-empty") + } + resp, err := sc.requireHTTP() + if err != nil { + return err + } + value, err := dsl.JSONField(resp.Body, path) + if err != nil { + return err + } + if strings.TrimSpace(value) == "" { + return fmt.Errorf("export to %s: json field %q is empty", name, path) + } + if err := sc.Suite.EnvLedger.Snapshot(name); err != nil { + return fmt.Errorf("export to %s: snapshot: %w", name, err) + } + if err := os.Setenv(name, value); err != nil { + return fmt.Errorf("export to %s: setenv: %w", name, err) + } + return nil +} + +func truncateBody(body string) string { + if len(body) <= maxHTTPFailureBody { + return body + } + return body[:maxHTTPFailureBody] + "..." + strconv.Itoa(len(body)) + " bytes" +} diff --git a/tests/bdd/steps/http_steps_test.go b/tests/bdd/steps/http_steps_test.go new file mode 100644 index 0000000000..79469012b7 --- /dev/null +++ b/tests/bdd/steps/http_steps_test.go @@ -0,0 +1,224 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package steps + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "nvcf-bdd/harness" +) + +type fakeHTTP struct { + responses []harness.HTTPResponse + err error + calls int + method string + url string + headers map[string]string + body string +} + +func (f *fakeHTTP) Do(_ context.Context, method, url string, headers map[string]string, body string) (harness.HTTPResponse, error) { + f.method, f.url, f.headers, f.body = method, url, headers, body + i := f.calls + f.calls++ + if f.err != nil { + return harness.HTTPResponse{}, f.err + } + if i >= len(f.responses) { + i = len(f.responses) - 1 + } + return f.responses[i], nil +} + +func newHTTPScenario(t *testing.T, responses ...harness.HTTPResponse) (*ScenarioContext, *fakeHTTP) { + t.Helper() + sc, _ := newScenarioContext(t) + fake := &fakeHTTP{responses: responses} + sc.Suite.HTTP = fake + return sc, fake +} + +func TestSendHTTPAppliesPendingHeadersOnceAndInterpolates(t *testing.T) { + t.Setenv("BDD_HTTP_HOST", "events.localhost:8080") + sc, fake := newHTTPScenario(t, harness.HTTPResponse{Status: 200, Body: "{}"}) + if err := sc.theHTTPRequestHeaderIs("Authorization", "Bearer ${BDD_HTTP_HOST}"); err != nil { + t.Fatal(err) + } + if err := sc.iSendHTTPRequest(context.Background(), "GET", "http://${BDD_HTTP_HOST}/health"); err != nil { + t.Fatal(err) + } + if fake.url != "http://events.localhost:8080/health" || fake.headers["Authorization"] != "Bearer events.localhost:8080" { + t.Fatalf("url=%q headers=%v", fake.url, fake.headers) + } + if err := sc.iSendHTTPRequest(context.Background(), "GET", "http://x/"); err != nil { + t.Fatal(err) + } + if len(fake.headers) != 0 { + t.Fatalf("headers leaked to next request: %v", fake.headers) + } +} + +func TestHTTPStatusAssertion(t *testing.T) { + sc, _ := newHTTPScenario(t, harness.HTTPResponse{Status: 401, Body: "nope"}) + if err := sc.theHTTPStatusShouldBe(401); err == nil { + t.Fatal("assertion must fail before any request") + } + if err := sc.iSendHTTPRequest(context.Background(), "POST", "http://x/"); err != nil { + t.Fatal(err) + } + if err := sc.theHTTPStatusShouldBe(401); err != nil { + t.Fatalf("401: %v", err) + } + if err := sc.theHTTPStatusShouldBe(200); err == nil || !strings.Contains(err.Error(), "nope") { + t.Fatalf("expected status mismatch with body, got %v", err) + } +} + +func TestHTTPStatusOneOfAssertion(t *testing.T) { + sc, _ := newHTTPScenario(t, harness.HTTPResponse{Status: 403, Body: "forbidden"}) + if err := sc.iSendHTTPRequest(context.Background(), "POST", "http://x/"); err != nil { + t.Fatal(err) + } + if err := sc.theHTTPStatusShouldBeOneOf(401, 403); err != nil { + t.Fatalf("403 should satisfy 401 or 403: %v", err) + } + if err := sc.theHTTPStatusShouldBeOneOf(200, 201); err == nil { + t.Fatal("expected mismatch") + } +} + +func TestHTTPSendErrorClearsLastResponse(t *testing.T) { + sc, fake := newHTTPScenario(t, harness.HTTPResponse{Status: 200}) + if err := sc.iSendHTTPRequest(context.Background(), "GET", "http://x/"); err != nil { + t.Fatal(err) + } + fake.err = errors.New("connection refused") + if err := sc.iSendHTTPRequest(context.Background(), "GET", "http://x/"); err == nil { + t.Fatal("expected send error") + } + if err := sc.theHTTPStatusShouldBe(200); err == nil { + t.Fatal("stale response must not satisfy assertion") + } +} + +func TestHTTPBodyAndJSONAssertions(t *testing.T) { + sc, _ := newHTTPScenario(t, harness.HTTPResponse{Status: 200, Body: `{"summary":{"by_event":{"InstanceReady":2}},"events":[{"source":"nvidia-spot"}]}`}) + if err := sc.iSendHTTPRequest(context.Background(), "GET", "http://x/"); err != nil { + t.Fatal(err) + } + if err := sc.theHTTPBodyShouldContain("InstanceReady"); err != nil { + t.Fatal(err) + } + if err := sc.theHTTPBodyShouldNotContain("InstanceDestroyed"); err != nil { + t.Fatal(err) + } + if err := sc.theHTTPBodyShouldNotContain("InstanceReady"); err == nil { + t.Fatal("not-contain must fail when present") + } + if err := sc.theHTTPBodyShouldContain(" "); err == nil { + t.Fatal("blank text must be rejected") + } + if err := sc.theHTTPJSONFieldShouldEqual("events.0.source", "nvidia-spot"); err != nil { + t.Fatal(err) + } + if err := sc.theHTTPJSONFieldShouldEqual("events.0.source", "nvca"); err == nil { + t.Fatal("expected mismatch") + } + if err := sc.theHTTPJSONFieldShouldBeAtLeast("summary.by_event.InstanceReady", 1); err != nil { + t.Fatal(err) + } +} + +func TestExportHTTPJSONFieldSetsEnvAndRestores(t *testing.T) { + sc, _ := newHTTPScenario(t, harness.HTTPResponse{Status: 200, Body: `{"value":"nvapi-abc"}`}) + t.Setenv("BDD_EXPORT_KEY", "before") + if err := sc.iSendHTTPRequest(context.Background(), "POST", "http://x/"); err != nil { + t.Fatal(err) + } + if err := sc.iExportHTTPJSONField("value", "BDD_EXPORT_KEY"); err != nil { + t.Fatal(err) + } + if got := os.Getenv("BDD_EXPORT_KEY"); got != "nvapi-abc" { + t.Fatalf("env = %q", got) + } + if err := sc.Suite.EnvLedger.RestoreAll(); err != nil { + t.Fatal(err) + } + if got := os.Getenv("BDD_EXPORT_KEY"); got != "before" { + t.Fatalf("env after restore = %q", got) + } + if err := sc.iExportHTTPJSONField("missing", "BDD_EXPORT_KEY"); err == nil { + t.Fatal("missing field must fail") + } +} + +func TestPollHTTPRetriesUntilBodyContains(t *testing.T) { + old := httpPollInterval + httpPollInterval = time.Millisecond + t.Cleanup(func() { httpPollInterval = old }) + sc, fake := newHTTPScenario(t, + harness.HTTPResponse{Status: 200, Body: `{"events":[]}`}, + harness.HTTPResponse{Status: 200, Body: `{"events":[{"event_name":"InstanceReady"}]}`}, + ) + if err := sc.iPollHTTPUntilBodyContains(context.Background(), "GET", "http://x/", 5, "InstanceReady"); err != nil { + t.Fatal(err) + } + if fake.calls != 2 { + t.Fatalf("calls = %d, want 2", fake.calls) + } +} + +func TestPollHTTPTimesOut(t *testing.T) { + old := httpPollInterval + httpPollInterval = 400 * time.Millisecond + t.Cleanup(func() { httpPollInterval = old }) + sc, _ := newHTTPScenario(t, harness.HTTPResponse{Status: 200, Body: `{"events":[]}`}) + err := sc.iPollHTTPUntilBodyContains(context.Background(), "GET", "http://x/", 1, "InstanceReady") + if err == nil || !strings.Contains(err.Error(), "timed out") { + t.Fatalf("expected timeout, got %v", err) + } +} + +func TestAdminTokenHeaderReadsStateFile(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + if err := os.WriteFile(filepath.Join(home, ".nvcf-cli.nvcf-cli-local.state"), []byte(`{"token":"jwt-1"}`), 0o600); err != nil { + t.Fatal(err) + } + sc, fake := newHTTPScenario(t, harness.HTTPResponse{Status: 200}) + if err := sc.theHTTPRequestHeaderIsAdminToken("Authorization", "Bearer "); err == nil { + t.Fatal("must require a selected CLI config") + } + sc.NVCFCLIConfig = "/repo/tests/bdd/fixtures/nvcf-cli-local.yaml" + if err := sc.theHTTPRequestHeaderIsAdminToken("Authorization", "Bearer "); err != nil { + t.Fatal(err) + } + if err := sc.iSendHTTPRequest(context.Background(), "GET", "http://x/"); err != nil { + t.Fatal(err) + } + if fake.headers["Authorization"] != "Bearer jwt-1" { + t.Fatalf("headers = %v", fake.headers) + } +} diff --git a/tests/bdd/steps/nvcf_cli_steps.go b/tests/bdd/steps/nvcf_cli_steps.go index e34efea48c..b7285bc97a 100644 --- a/tests/bdd/steps/nvcf_cli_steps.go +++ b/tests/bdd/steps/nvcf_cli_steps.go @@ -258,7 +258,11 @@ func isTransientVanityRouteNotFound(stderr string) bool { } func (sc *ScenarioContext) iSuccessfullyUndeploySelectedFunction(ctx context.Context) error { - return sc.runNVCFCLI(ctx, "function", "delete", "--deployment-only") + if err := sc.runNVCFCLI(ctx, "function", "delete", "--deployment-only"); err != nil { + return err + } + sc.FunctionUndeployed = true + return nil } func (sc *ScenarioContext) runNVCFCLIWithOptions( diff --git a/tests/bdd/steps/teardown_steps.go b/tests/bdd/steps/teardown_steps.go new file mode 100644 index 0000000000..3c070ceddf --- /dev/null +++ b/tests/bdd/steps/teardown_steps.go @@ -0,0 +1,114 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package steps + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "github.com/cucumber/godog" + + "nvcf-bdd/dsl" +) + +// functionDeploymentNotFound is how the NVCF API reports a function with no +// deployment, which is the expected state when a scenario already undeployed it. +const functionDeploymentNotFound = "Deployment not found" + +// registerTeardownSteps hooks the steps that register cleanup to run after the +// scenario ends, whether it passed or failed. The live runner stops at the +// first failed step, so cleanup written as a later step would never run. +func registerTeardownSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { + ctx.Step(`^I send an? "([A-Z]+)" request to "([^"]*)" at scenario teardown expecting status (\d+)$`, sc.iSendHTTPRequestAtTeardown) + ctx.Step(`^the function selected by NVCF CLI is undeployed at scenario teardown$`, sc.theSelectedFunctionIsUndeployedAtTeardown) +} + +// addTeardown queues cleanup for the end of the current scenario. +func (sc *ScenarioContext) addTeardown(description string, action func(context.Context) error) { + sc.teardown = append(sc.teardown, teardownAction{description: description, run: action}) +} + +// iSendHTTPRequestAtTeardown queues a request for the end of the scenario. It +// takes the headers queued so far, like the send step does, so a credential is +// declared next to the request that needs it. The URL is resolved now, so any +// variable it uses must already be exported. Any status other than the +// expected one fails the scenario. +func (sc *ScenarioContext) iSendHTTPRequestAtTeardown(method, url string, wantStatus int) error { + headers := sc.HTTPHeaders + sc.HTTPHeaders = nil + resolved := dsl.Interpolate(url) + sc.addTeardown(method+" "+resolved, func(ctx context.Context) error { + resp, err := sc.httpClient().Do(ctx, method, resolved, headers, "") + if err != nil { + return err + } + if resp.Status != wantStatus { + return fmt.Errorf("status %d, want %d: %s", resp.Status, wantStatus, truncateBody(resp.Body)) + } + return nil + }) + return nil +} + +// theSelectedFunctionIsUndeployedAtTeardown queues an undeploy of the function +// the NVCF CLI has selected. It does nothing when the scenario already +// undeployed the function, and treats a function with no deployment as done. +func (sc *ScenarioContext) theSelectedFunctionIsUndeployedAtTeardown() error { + if sc.NVCFCLIConfig == "" { + return fmt.Errorf("no NVCF CLI config selected; use 'I use NVCF CLI config' first") + } + command := dsl.BuildCommand(sc.nvcfCLICommandArgs("function", "delete", "--deployment-only")...) + sc.addTeardown("undeploy the selected function", func(ctx context.Context) error { + if sc.FunctionUndeployed { + return nil + } + result, err := sc.Suite.Runner.Run(ctx, command) + if err == nil && result.ExitCode == 0 { + return nil + } + output := combinedOutput(result) + if strings.Contains(output, functionDeploymentNotFound) { + return nil + } + if err != nil { + return fmt.Errorf("exit code %d: %w", result.ExitCode, err) + } + return fmt.Errorf("exit code %d", result.ExitCode) + }) + return nil +} + +// runTeardown runs the queued cleanup in reverse order, so a resource is +// released before the one it depends on. A failing action does not stop the +// rest. The queue is cleared and every failure is reported together. +func (sc *ScenarioContext) runTeardown(ctx context.Context) error { + actions := sc.teardown + sc.teardown = nil + var failures []error + for i := len(actions) - 1; i >= 0; i-- { + action := actions[i] + fmt.Fprintf(os.Stderr, ">>> teardown: %s\n", action.description) + if err := action.run(ctx); err != nil { + failures = append(failures, fmt.Errorf("teardown %q: %w", action.description, err)) + } + } + return errors.Join(failures...) +} diff --git a/tests/bdd/steps/teardown_steps_test.go b/tests/bdd/steps/teardown_steps_test.go new file mode 100644 index 0000000000..e097b9e666 --- /dev/null +++ b/tests/bdd/steps/teardown_steps_test.go @@ -0,0 +1,175 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package steps + +import ( + "context" + "errors" + "strings" + "testing" + + "nvcf-bdd/harness" +) + +func TestHTTPTeardownSendsRegisteredRequestWithItsHeaders(t *testing.T) { + t.Setenv("BDD_TEARDOWN_KEY_ID", "key-1") + sc, fake := newHTTPScenario(t, harness.HTTPResponse{Status: 204}) + if err := sc.theHTTPRequestHeaderIs("Key-Owner-Id", "owner"); err != nil { + t.Fatal(err) + } + if err := sc.iSendHTTPRequestAtTeardown("DELETE", "http://api-keys/v1/keys/${BDD_TEARDOWN_KEY_ID}", 204); err != nil { + t.Fatal(err) + } + if fake.calls != 0 { + t.Fatalf("request sent at registration: %d calls", fake.calls) + } + if len(sc.HTTPHeaders) != 0 { + t.Fatalf("pending headers not consumed: %v", sc.HTTPHeaders) + } + + if err := sc.runTeardown(context.Background()); err != nil { + t.Fatalf("teardown: %v", err) + } + if fake.method != "DELETE" || fake.url != "http://api-keys/v1/keys/key-1" || fake.headers["Key-Owner-Id"] != "owner" { + t.Fatalf("sent %s %s headers=%v", fake.method, fake.url, fake.headers) + } +} + +func TestHTTPTeardownReportsFailure(t *testing.T) { + sc, _ := newHTTPScenario(t, harness.HTTPResponse{Status: 500, Body: "boom"}) + if err := sc.iSendHTTPRequestAtTeardown("DELETE", "http://x/", 204); err != nil { + t.Fatal(err) + } + err := sc.runTeardown(context.Background()) + if err == nil || !strings.Contains(err.Error(), "status 500, want 204") { + t.Fatalf("expected status failure, got %v", err) + } +} + +func TestHTTPTeardownRequiresTheExactStatus(t *testing.T) { + sc, _ := newHTTPScenario(t, harness.HTTPResponse{Status: 200}) + if err := sc.iSendHTTPRequestAtTeardown("DELETE", "http://x/", 204); err != nil { + t.Fatal(err) + } + if err := sc.runTeardown(context.Background()); err == nil || !strings.Contains(err.Error(), "status 200, want 204") { + t.Fatalf("a 200 must not satisfy an expected 204, got %v", err) + } +} + +func TestTeardownRunsInReverseOrderAndContinuesAfterFailure(t *testing.T) { + sc, _ := newScenarioContext(t) + var order []string + for _, name := range []string{"first", "second", "third"} { + sc.addTeardown(name, func(context.Context) error { + order = append(order, name) + if name == "second" { + return errors.New("second failed") + } + return nil + }) + } + err := sc.runTeardown(context.Background()) + if strings.Join(order, ",") != "third,second,first" { + t.Fatalf("order = %v", order) + } + if err == nil || !strings.Contains(err.Error(), "second failed") { + t.Fatalf("expected the second failure, got %v", err) + } + if len(sc.teardown) != 0 { + t.Fatal("queue not cleared") + } + if err := sc.runTeardown(context.Background()); err != nil { + t.Fatalf("second run should be a no-op: %v", err) + } +} + +func TestUndeployTeardownRequiresSelectedConfig(t *testing.T) { + sc, _ := newScenarioContext(t) + if err := sc.theSelectedFunctionIsUndeployedAtTeardown(); err == nil { + t.Fatal("expected an error without an NVCF CLI config") + } +} + +func TestUndeployTeardownRunsTheCLIOnce(t *testing.T) { + t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") + sc, fake := newScenarioContext(t) + sc.NVCFCLIConfig = "/cfg.yaml" + if err := sc.theSelectedFunctionIsUndeployedAtTeardown(); err != nil { + t.Fatal(err) + } + if len(fake.runs) != 0 { + t.Fatal("undeploy ran at registration") + } + if err := sc.runTeardown(context.Background()); err != nil { + t.Fatal(err) + } + if len(fake.runs) != 1 || !strings.Contains(fake.runs[0].command, "function delete --deployment-only") || + !strings.Contains(fake.runs[0].command, "--config /cfg.yaml") { + t.Fatalf("runs = %+v", fake.runs) + } +} + +func TestUndeployTeardownSkipsAfterTheScenarioUndeployed(t *testing.T) { + t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") + sc, fake := newScenarioContext(t) + sc.NVCFCLIConfig = "/cfg.yaml" + if err := sc.theSelectedFunctionIsUndeployedAtTeardown(); err != nil { + t.Fatal(err) + } + if err := sc.iSuccessfullyUndeploySelectedFunction(context.Background()); err != nil { + t.Fatal(err) + } + if !sc.FunctionUndeployed { + t.Fatal("successful undeploy not remembered") + } + runsBefore := len(fake.runs) + if err := sc.runTeardown(context.Background()); err != nil { + t.Fatal(err) + } + if len(fake.runs) != runsBefore { + t.Fatalf("teardown undeployed again: %+v", fake.runs[runsBefore:]) + } +} + +func TestUndeployTeardownToleratesAFunctionWithNoDeployment(t *testing.T) { + t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") + sc, fake := newScenarioContext(t) + sc.NVCFCLIConfig = "/cfg.yaml" + fake.result = harness.Result{ExitCode: 1, Stderr: `Error: API error 404: {"detail":"Deployment not found"}`} + fake.err = errors.New("exit status 1") + if err := sc.theSelectedFunctionIsUndeployedAtTeardown(); err != nil { + t.Fatal(err) + } + if err := sc.runTeardown(context.Background()); err != nil { + t.Fatalf("not-found should count as already undeployed: %v", err) + } +} + +func TestUndeployTeardownReportsOtherFailures(t *testing.T) { + t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") + sc, fake := newScenarioContext(t) + sc.NVCFCLIConfig = "/cfg.yaml" + fake.result = harness.Result{ExitCode: 1, Stderr: "Error: connection refused"} + fake.err = errors.New("exit status 1") + if err := sc.theSelectedFunctionIsUndeployedAtTeardown(); err != nil { + t.Fatal(err) + } + if err := sc.runTeardown(context.Background()); err == nil { + t.Fatal("expected the failure to be reported") + } +}