test(bdd): add event-ledger live checks and HTTP request steps

deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl

@@ -285,7 +285,7 @@ releases:
 
   - name: event-ledger
     chart: nvcf/helm-nvcf-event-ledger
-    version: 0.5.0
+    version: 0.6.0
     namespace: nvcf
     condition: addons.eventLedger.enabled
     values:

tests/bdd/PLAN.md

@@ -174,7 +174,35 @@ original order. Repeated options and empty values are preserved.
 | `When I successfully invoke the function selected by NVCF CLI over plaintext gRPC service {string} method {string} with timeout {string} seconds and poll duration {string} seconds:` (JSON docstring) | Runs `function invoke --grpc --grpc-plaintext` with the visible service, method, request, timeout, and poll duration. |
 | `When I successfully invoke model {string} at {string} with timeout {string} seconds:` (JSON docstring) | Runs `function invoke` with the visible model, inference URL, exact request body, and timeout. |
 | `When I successfully invoke the function selected by NVCF CLI through Vanity Gateway host {string} path {string} with timeout {string} seconds:` (JSON docstring) | Sends an exact-host HTTP request through the local Envoy listener with the saved function API key passed over sensitive stdin, never argv or command logs. |
-| `When I successfully undeploy the function selected by NVCF CLI` | Runs `function delete --deployment-only`. Function selection remains owned by CLI state. |
+| `When I successfully undeploy the function selected by NVCF CLI` | Runs `function delete --deployment-only`. Function selection remains owned by CLI state. A successful run is remembered, so a queued teardown undeploy skips it. |
+
+### HTTP requests (Given / When / Then)
+
+These steps use the in-process HTTP client (`harness.HTTPClient`), not the command runner, so headers never reach `out/<run-id>/logs`. Redirects are not followed. Pending headers apply to the next send or poll step only.
+
+| Step | Notes |
+|------|-------|
+| `Given the HTTP request header {string} is {string}` | Queues one request header for the next request. `${VAR}` expansion applies to the value. |
+| `Given the HTTP request header {string} is the NVCF CLI admin token with prefix {string}` | Queues a header whose value is the prefix plus the admin token stored by `nvcf-cli init` in the state file of the selected NVCF CLI config. The token never enters argv, env, or logs. |
+| `When I send a {string} request to {string}` | Sends the request with the queued headers and records status and body. `${VAR}` expansion applies to the URL. A non-2xx status is not a step failure. |
+| `When I send a {string} request to {string} with body:` (docstring) | As above with the interpolated docstring as the request body. |
+| `When I poll {string} {string} for up to {int} seconds until the HTTP response body contains {string}` | Repeats the request every 5 seconds until the body contains the text or the deadline passes. Used for eventually consistent reads. |
+| `Then the HTTP status should be {int}` | Last response status. The failure message includes a truncated body. |
+| `Then the HTTP status should be {int} or {int}` | Last response status equals either value. |
+| `Then the HTTP response body should contain {string}` | Substring match. The interpolated value must be non-empty. |
+| `Then the HTTP response body should not contain {string}` | Negative substring match. The interpolated value must be non-empty. |
+| `Then the HTTP response JSON field {string} should equal {string}` | Dotted path (object keys, array indexes) compared as text. |
+| `Then the HTTP response JSON field {string} should be at least {int}` | Dotted path parsed as a number and compared to a minimum. |
+| `When I export the HTTP response JSON field {string} to environment variable {string}` | Exports the field under the named env var. Fails when the field is missing or empty. Snapshotted by the env Ledger; restored at suite teardown. The value is never echoed. |
+
+### Scenario teardown (Given / When)
+
+Cleanup written as a later step is skipped when an earlier step fails, because the live runner stops at the first failed step. These steps queue cleanup that runs after the scenario ends instead, whether it passed or failed. Queued actions run in reverse order through a Godog After hook, a failing action does not stop the rest, and every failure is reported by failing the scenario. The queue is cleared before each scenario.
+
+| Step | Notes |
+|------|-------|
+| `When I send a {string} request to {string} at scenario teardown expecting status {int}` | Queues the request instead of sending it now. It takes the queued headers like the send step, so declare a credential just before it. `${VAR}` expansion applies to the URL when the step runs, so the variable must already be exported. Any status other than the expected one fails the scenario. |
+| `When the function selected by NVCF CLI is undeployed at scenario teardown` | Queues `function delete --deployment-only` for the selected CLI config. It does nothing when the scenario already undeployed the function with the undeploy step. A "Deployment not found" result counts as done, and any other failure fails the scenario. |
 
 ### Assertions (Then / And)
 

tests/bdd/dsl/clistate.go

@@ -0,0 +1,52 @@
+/*
+SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dsl
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// CLIStatePath returns the nvcf-cli state file for a --config path, mirroring
+// the CLI: ~/.nvcf-cli.<config basename without extension>.state.
+func CLIStatePath(home, configPath string) string {
+	name := strings.TrimSuffix(filepath.Base(configPath), filepath.Ext(configPath))
+	return filepath.Join(home, fmt.Sprintf(".nvcf-cli.%s.state", name))
+}
+
+// CLIStateToken reads the admin token that `nvcf-cli init` stored in the
+// state file at path. It fails when the file or the token is missing.
+func CLIStateToken(path string) (string, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return "", fmt.Errorf("read nvcf-cli state: %w", err)
+	}
+	var state struct {
+		Token string `json:"token"`
+	}
+	if err := json.Unmarshal(data, &state); err != nil {
+		return "", fmt.Errorf("parse nvcf-cli state %s: %w", path, err)
+	}
+	if state.Token == "" {
+		return "", fmt.Errorf("nvcf-cli state %s has no admin token; run nvcf-cli init first", path)
+	}
+	return state.Token, nil
+}

tests/bdd/dsl/http_test.go

@@ -0,0 +1,107 @@
+/*
+SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dsl
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+const ledgerEvents = `{"namespace":"ns","summary":{"total_contexts":2,"by_event":{"InstanceReady":2}},"events":[{"event_name":"InstanceReady","source":"nvidia-spot"}]}`
+
+func TestJSONFieldReadsNestedValues(t *testing.T) {
+	cases := map[string]string{
+		"namespace":                      "ns",
+		"summary.total_contexts":         "2",
+		"summary.by_event.InstanceReady": "2",
+		"events.0.event_name":            "InstanceReady",
+		"events.0.source":                "nvidia-spot",
+	}
+	for path, want := range cases {
+		got, err := JSONField(ledgerEvents, path)
+		if err != nil {
+			t.Fatalf("%s: %v", path, err)
+		}
+		if got != want {
+			t.Fatalf("%s = %q, want %q", path, got, want)
+		}
+	}
+}
+
+func TestJSONFieldReturnsCompactJSONForObjects(t *testing.T) {
+	got, err := JSONField(ledgerEvents, "summary.by_event")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != `{"InstanceReady":2}` {
+		t.Fatalf("got %q", got)
+	}
+}
+
+func TestJSONFieldErrors(t *testing.T) {
+	for _, path := range []string{"", "missing", "events.5", "events.x", "namespace.child"} {
+		if _, err := JSONField(ledgerEvents, path); err == nil {
+			t.Fatalf("path %q: expected error", path)
+		}
+	}
+	if _, err := JSONField("not json", "a"); err == nil {
+		t.Fatal("expected parse error")
+	}
+}
+
+func TestJSONFieldAtLeast(t *testing.T) {
+	if err := JSONFieldAtLeast(ledgerEvents, "summary.by_event.InstanceReady", 1); err != nil {
+		t.Fatalf("at least 1: %v", err)
+	}
+	if err := JSONFieldAtLeast(ledgerEvents, "summary.by_event.InstanceReady", 3); err == nil {
+		t.Fatal("expected failure for min 3")
+	}
+	if err := JSONFieldAtLeast(ledgerEvents, "namespace", 1); err == nil {
+		t.Fatal("expected failure for non-numeric field")
+	}
+}
+
+func TestCLIStatePathMirrorsCLI(t *testing.T) {
+	got := CLIStatePath("/home/u", "/repo/tests/bdd/fixtures/nvcf-cli-local.yaml")
+	if got != "/home/u/.nvcf-cli.nvcf-cli-local.state" {
+		t.Fatalf("got %q", got)
+	}
+}
+
+func TestCLIStateToken(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "state")
+	if err := os.WriteFile(path, []byte(`{"token":"abc"}`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	got, err := CLIStateToken(path)
+	if err != nil || got != "abc" {
+		t.Fatalf("token = %q, err = %v", got, err)
+	}
+	if err := os.WriteFile(path, []byte(`{}`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := CLIStateToken(path); err == nil || !strings.Contains(err.Error(), "nvcf-cli init") {
+		t.Fatalf("expected missing-token error, got %v", err)
+	}
+	if _, err := CLIStateToken(filepath.Join(dir, "absent")); err == nil {
+		t.Fatal("expected read error")
+	}
+}

tests/bdd/dsl/jsonfield.go

@@ -0,0 +1,100 @@
+/*
+SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dsl
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// JSONField reads the dotted path from a JSON document and returns the value
+// as text. Object keys are matched by name and array elements by decimal
+// index, so "events.0.event_name" reads the first event. Scalars come back as
+// their plain text; objects and arrays come back as compact JSON.
+func JSONField(raw, path string) (string, error) {
+	if strings.TrimSpace(path) == "" {
+		return "", fmt.Errorf("json path must be non-empty")
+	}
+	var node any
+	if err := json.Unmarshal([]byte(raw), &node); err != nil {
+		return "", fmt.Errorf("parse json: %w", err)
+	}
+	for _, part := range strings.Split(path, ".") {
+		next, err := childNode(node, part)
+		if err != nil {
+			return "", fmt.Errorf("json path %q: %w", path, err)
+		}
+		node = next
+	}
+	return nodeText(node)
+}
+
+// JSONFieldAtLeast reports whether the numeric value at the dotted path is
+// greater than or equal to min.
+func JSONFieldAtLeast(raw, path string, min float64) error {
+	text, err := JSONField(raw, path)
+	if err != nil {
+		return err
+	}
+	value, err := strconv.ParseFloat(text, 64)
+	if err != nil {
+		return fmt.Errorf("json path %q is not a number: %q", path, text)
+	}
+	if value < min {
+		return fmt.Errorf("json path %q = %v, want at least %v", path, value, min)
+	}
+	return nil
+}
+
+func childNode(node any, part string) (any, error) {
+	switch typed := node.(type) {
+	case map[string]any:
+		child, ok := typed[part]
+		if !ok {
+			return nil, fmt.Errorf("key %q not found", part)
+		}
+		return child, nil
+	case []any:
+		index, err := strconv.Atoi(part)
+		if err != nil || index < 0 || index >= len(typed) {
+			return nil, fmt.Errorf("index %q out of range for array of %d", part, len(typed))
+		}
+		return typed[index], nil
+	default:
+		return nil, fmt.Errorf("cannot descend into %q: value is not an object or array", part)
+	}
+}
+
+func nodeText(node any) (string, error) {
+	switch typed := node.(type) {
+	case string:
+		return typed, nil
+	case nil:
+		return "null", nil
+	case map[string]any, []any:
+		data, err := json.Marshal(typed)
+		if err != nil {
+			return "", fmt.Errorf("encode json: %w", err)
+		}
+		return string(data), nil
+	default:
+		return fmt.Sprint(typed), nil
+	}
+}