From 7143d3c98858f701eb24af1c7db82accfcdbc88b Mon Sep 17 00:00:00 2001 From: Dan Mace Date: Thu, 16 Jul 2026 12:42:36 -0400 Subject: [PATCH] feat(e2e/v2): make lifecycle binaries configurable for non-CI environments Remove hard-coded assumptions from the v2 lifecycle binaries so they can be driven by external tooling outside of CI step registry jobs. - Add variant filtering, namespace, base domain, binary path, and label filter override via environment variables - Pass --namespace to create/destroy CLI commands - Support deterministic infra IDs via HYPERSHIFT_INFRA_ID_FROM_NAME - Add FilterClusterSpecs and FilterTestMatrix helpers to lifecycle pkg --- test/e2e/v2/cmd/create-guests/main.go | 16 +- test/e2e/v2/cmd/destroy-guests/main.go | 23 ++- test/e2e/v2/cmd/dump-guests/main.go | 9 +- test/e2e/v2/cmd/run-tests/main.go | 36 ++++- test/e2e/v2/lifecycle/platform.go | 54 ++++++- test/e2e/v2/lifecycle/platform_test.go | 200 +++++++++++++++++++++++++ 6 files changed, 319 insertions(+), 19 deletions(-) create mode 100644 test/e2e/v2/lifecycle/platform_test.go diff --git a/test/e2e/v2/cmd/create-guests/main.go b/test/e2e/v2/cmd/create-guests/main.go index c3be0061dea6..e93da60ae033 100644 --- a/test/e2e/v2/cmd/create-guests/main.go +++ b/test/e2e/v2/cmd/create-guests/main.go @@ -62,8 +62,6 @@ func init() { utilruntime.Must(routev1.AddToScheme(scheme)) } -const defaultNamespace = "clusters" - // envConfig captures the common environment configuration. type envConfig struct { prowJobID string @@ -79,6 +77,9 @@ type envConfig struct { etcdSC string pullSecret string + variants string + infraIDFromName bool + platform lifecycle.PlatformConfig hypershiftBinary string waitTimeout time.Duration @@ -112,11 +113,14 @@ func loadEnvConfig() envConfig { baseDomain: envOrDefault("HYPERSHIFT_BASE_DOMAIN", platform.DefaultBaseDomain()), nodeCount: envOrDefaultInt("HYPERSHIFT_NODE_COUNT", 3), - namespace: envOrDefault("HYPERSHIFT_NAMESPACE", defaultNamespace), + namespace: envOrDefault("HYPERSHIFT_NAMESPACE", lifecycle.DefaultNamespace), externalDNS: os.Getenv("HYPERSHIFT_EXTERNAL_DNS_DOMAIN"), etcdSC: os.Getenv("HYPERSHIFT_ETCD_STORAGE_CLASS"), pullSecret: envOrDefault("PULL_SECRET", "/etc/ci-pull-credentials/.dockerconfigjson"), + variants: os.Getenv("HYPERSHIFT_VARIANTS"), + infraIDFromName: os.Getenv("HYPERSHIFT_INFRA_ID_FROM_NAME") == "true", + platform: platform, hypershiftBinary: envOrDefault("HYPERSHIFT_BINARY", "hypershift"), waitTimeout: 45 * time.Minute, @@ -130,7 +134,7 @@ func loadEnvConfig() envConfig { } func run(ctx context.Context, cfg envConfig) error { - specs := cfg.platform.ClusterSpecs(cfg.releaseImage, cfg.n1Image) + specs := lifecycle.FilterClusterSpecs(cfg.platform.ClusterSpecs(cfg.releaseImage, cfg.n1Image), cfg.variants) // Derive cluster names and build the name map. named := make([]namedSpec, len(specs)) @@ -251,6 +255,7 @@ func buildCreateArgs(cfg envConfig, name string, spec lifecycle.ClusterSpec) []s args := []string{ "create", "cluster", cfg.platform.Name(), "--name=" + name, + "--namespace=" + cfg.namespace, "--node-pool-replicas=" + strconv.Itoa(cfg.nodeCount), "--base-domain=" + cfg.baseDomain, "--pull-secret=" + cfg.pullSecret, @@ -258,6 +263,9 @@ func buildCreateArgs(cfg envConfig, name string, spec lifecycle.ClusterSpec) []s "--generate-ssh", } + if cfg.infraIDFromName { + args = append(args, "--infra-id="+name) + } if cfg.externalDNS != "" { args = append(args, "--external-dns-domain="+cfg.externalDNS) } diff --git a/test/e2e/v2/cmd/destroy-guests/main.go b/test/e2e/v2/cmd/destroy-guests/main.go index 5f317b16ab50..1950bd28373f 100644 --- a/test/e2e/v2/cmd/destroy-guests/main.go +++ b/test/e2e/v2/cmd/destroy-guests/main.go @@ -52,7 +52,15 @@ func main() { hypershiftBin = "hypershift" } - specs := platform.ClusterSpecs("", "") + namespace := os.Getenv("HYPERSHIFT_NAMESPACE") + if namespace == "" { + namespace = lifecycle.DefaultNamespace + } + + baseDomain := os.Getenv("HYPERSHIFT_BASE_DOMAIN") + infraIDFromName := os.Getenv("HYPERSHIFT_INFRA_ID_FROM_NAME") == "true" + + specs := lifecycle.FilterClusterSpecs(platform.ClusterSpecs("", ""), os.Getenv("HYPERSHIFT_VARIANTS")) log.Printf("Destroying %d clusters derived from PROW_JOB_ID=%s", len(specs), prowJobID) @@ -67,7 +75,7 @@ func main() { wg.Add(1) go func() { defer wg.Done() - if err := destroyCluster(hypershiftBin, clusterName, spec.Variant, platform); err != nil { + if err := destroyCluster(hypershiftBin, clusterName, namespace, baseDomain, infraIDFromName, platform); err != nil { log.Printf("WARNING: Failed to destroy cluster %s (%s): %v", clusterName, spec.Variant, err) log.Printf("ACTION REQUIRED: cloud resources for cluster %s may be orphaned and need manual cleanup (resource group, DNS records, etc.)", clusterName) mu.Lock() @@ -85,14 +93,21 @@ func main() { log.Printf("All clusters destroyed successfully") } -func destroyCluster(hypershiftBin, name, variant string, platform lifecycle.PlatformConfig) error { - log.Printf("Destroying cluster %s (%s)", name, variant) +func destroyCluster(hypershiftBin, name, namespace, baseDomain string, infraIDFromName bool, platform lifecycle.PlatformConfig) error { + log.Printf("Destroying cluster %s", name) args := []string{ "destroy", "cluster", platform.Name(), "--name=" + name, + "--namespace=" + namespace, "--cluster-grace-period=" + clusterGracePeriod, } + if infraIDFromName { + args = append(args, "--infra-id="+name) + } + if baseDomain != "" { + args = append(args, "--base-domain="+baseDomain) + } args = append(args, platform.DestroyArgs()...) log.Printf("Running: %s %v", hypershiftBin, args) diff --git a/test/e2e/v2/cmd/dump-guests/main.go b/test/e2e/v2/cmd/dump-guests/main.go index 3f0fec53999d..ebb0b16656a1 100644 --- a/test/e2e/v2/cmd/dump-guests/main.go +++ b/test/e2e/v2/cmd/dump-guests/main.go @@ -34,7 +34,11 @@ import ( ) func main() { - hypershiftBinary := flag.String("hypershift-binary", "hypershift", "Path to the hypershift CLI binary") + defaultBinary := "hypershift" + if v := os.Getenv("HYPERSHIFT_BINARY"); v != "" { + defaultBinary = v + } + hypershiftBinary := flag.String("hypershift-binary", defaultBinary, "Path to the hypershift CLI binary") flag.Parse() prowJobID := os.Getenv("PROW_JOB_ID") @@ -52,7 +56,8 @@ func main() { log.Fatalf("Failed to initialize platform config: %v", err) } - specs := platform.ClusterSpecs("", "") + variants := os.Getenv("HYPERSHIFT_VARIANTS") + specs := lifecycle.FilterClusterSpecs(platform.ClusterSpecs("", ""), variants) log.Printf("Dumping %d clusters derived from PROW_JOB_ID=%s", len(specs), prowJobID) var wg sync.WaitGroup diff --git a/test/e2e/v2/cmd/run-tests/main.go b/test/e2e/v2/cmd/run-tests/main.go index 260b82a43593..2c58d0ee0268 100644 --- a/test/e2e/v2/cmd/run-tests/main.go +++ b/test/e2e/v2/cmd/run-tests/main.go @@ -18,9 +18,7 @@ import ( ) const ( - testBinary = "bin/test-e2e-v2" - clusterNS = "clusters" - defaultVerbose = "false" + defaultVerbose = "false" defaultGinkgoTimeout = "3h" ) @@ -37,6 +35,11 @@ func main() { artifactDir := requireEnv("ARTIFACT_DIR") releaseImage := os.Getenv("RELEASE_IMAGE_LATEST") + testBinary := "bin/test-e2e-v2" + if binDir := os.Getenv("E2EV2_BIN_DIR"); binDir != "" { + testBinary = filepath.Join(binDir, "test-e2e-v2") + } + eventuallyVerbose := os.Getenv("EVENTUALLY_VERBOSE") if eventuallyVerbose == "" { eventuallyVerbose = defaultVerbose @@ -51,7 +54,22 @@ func main() { // Let the platform set up any env vars it needs for tests. platform.SetupTestEnv(sharedDir) - matrix := platform.TestMatrix(releaseImage) + variants := os.Getenv("HYPERSHIFT_VARIANTS") + specs := lifecycle.FilterClusterSpecs(platform.ClusterSpecs(releaseImage, os.Getenv("OCP_IMAGE_N1")), variants) + matrix := lifecycle.FilterTestMatrix(platform.TestMatrix(releaseImage), specs) + + // Allow overriding the label filter for all test groups. + if override := os.Getenv("GINKGO_LABEL_FILTER"); override != "" { + log.Printf("Overriding label filters with GINKGO_LABEL_FILTER=%s", override) + for i := range matrix.Parallel { + matrix.Parallel[i].LabelFilter = override + } + for i := range matrix.Sequential { + for j := range matrix.Sequential[i].Steps { + matrix.Sequential[i].Steps[j].LabelFilter = override + } + } + } var ( mu sync.Mutex @@ -67,7 +85,7 @@ func main() { defer wg.Done() clusterName := readClusterName(sharedDir, g.ClusterFile) log.Printf("Running %s tests against %s...", g.Name, clusterName) - err := runTestBinary(clusterName, g.LabelFilter, g.Skip, + err := runTestBinary(testBinary, clusterName, g.LabelFilter, g.Skip, filepath.Join(artifactDir, g.JUnitFile), g.ExtraEnv) mu.Lock() results = append(results, testResult{name: g.Name, err: err}) @@ -90,7 +108,7 @@ func main() { for i, step := range sg.Steps { clusterName := readClusterName(sharedDir, step.ClusterFile) log.Printf("Running %s tests against %s...", step.Name, clusterName) - err := runTestBinary(clusterName, step.LabelFilter, step.Skip, + err := runTestBinary(testBinary, clusterName, step.LabelFilter, step.Skip, filepath.Join(artifactDir, step.JUnitFile), step.ExtraEnv) mu.Lock() results = append(results, testResult{name: step.Name, err: err}) @@ -126,7 +144,7 @@ func main() { log.Println("All test groups passed") } -func runTestBinary(clusterName, labelFilter, skip, junitPath string, extraEnv []string) error { +func runTestBinary(testBinary, clusterName, labelFilter, skip, junitPath string, extraEnv []string) error { ginkgoTimeout := os.Getenv("GINKGO_TIMEOUT") if ginkgoTimeout == "" { ginkgoTimeout = defaultGinkgoTimeout @@ -146,6 +164,10 @@ func runTestBinary(clusterName, labelFilter, skip, junitPath string, extraEnv [] cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr + clusterNS := os.Getenv("HYPERSHIFT_NAMESPACE") + if clusterNS == "" { + clusterNS = lifecycle.DefaultNamespace + } cmd.Env = append(os.Environ(), fmt.Sprintf("E2E_HOSTED_CLUSTER_NAME=%s", clusterName), fmt.Sprintf("E2E_HOSTED_CLUSTER_NAMESPACE=%s", clusterNS), diff --git a/test/e2e/v2/lifecycle/platform.go b/test/e2e/v2/lifecycle/platform.go index 737ea3f51f32..2b71c2504af8 100644 --- a/test/e2e/v2/lifecycle/platform.go +++ b/test/e2e/v2/lifecycle/platform.go @@ -6,10 +6,14 @@ import ( "context" "crypto/sha256" "fmt" + "strings" crclient "sigs.k8s.io/controller-runtime/pkg/client" ) +// DefaultNamespace is the default namespace for hosted clusters. +const DefaultNamespace = "clusters" + // ClusterSpec describes a single cluster to create for lifecycle tests. type ClusterSpec struct { Variant string @@ -95,7 +99,6 @@ type PlatformConfig interface { // DestroyArgs returns platform-specific args for // "hypershift destroy cluster ". DestroyArgs() []string - } // NewPlatformConfig creates a PlatformConfig for the given platform @@ -110,6 +113,54 @@ func NewPlatformConfig(platform, sharedDir string) (PlatformConfig, error) { } } +// FilterClusterSpecs returns only the specs whose Variant is in the +// comma-separated variants string. If variants is empty, all specs +// are returned. +func FilterClusterSpecs(specs []ClusterSpec, variants string) []ClusterSpec { + if variants == "" { + return specs + } + allowed := make(map[string]bool) + for _, v := range strings.Split(variants, ",") { + allowed[strings.TrimSpace(v)] = true + } + var filtered []ClusterSpec + for _, s := range specs { + if allowed[s.Variant] { + filtered = append(filtered, s) + } + } + return filtered +} + +// FilterTestMatrix removes test groups that reference cluster files +// not present in the given specs. +func FilterTestMatrix(matrix TestMatrix, specs []ClusterSpec) TestMatrix { + clusterFiles := make(map[string]bool) + for _, s := range specs { + clusterFiles[s.OutputFile] = true + } + var parallel []TestGroup + for _, g := range matrix.Parallel { + if clusterFiles[g.ClusterFile] { + parallel = append(parallel, g) + } + } + var sequential []SequentialGroup + for _, sg := range matrix.Sequential { + var steps []TestGroup + for _, step := range sg.Steps { + if clusterFiles[step.ClusterFile] { + steps = append(steps, step) + } + } + if len(steps) > 0 { + sequential = append(sequential, SequentialGroup{Name: sg.Name, Steps: steps}) + } + } + return TestMatrix{Parallel: parallel, Sequential: sequential} +} + // DeriveClusterName builds a human-readable, deterministic cluster name // from the prow job ID and cluster variant. The format is // "{variant}-{hash10}" where hash10 is the first 10 hex characters of @@ -119,4 +170,3 @@ func DeriveClusterName(prowJobID, variant string) string { hash := sha256.Sum256([]byte(prowJobID)) return variant + "-" + fmt.Sprintf("%x", hash)[:10] } - diff --git a/test/e2e/v2/lifecycle/platform_test.go b/test/e2e/v2/lifecycle/platform_test.go new file mode 100644 index 000000000000..c56451ced767 --- /dev/null +++ b/test/e2e/v2/lifecycle/platform_test.go @@ -0,0 +1,200 @@ +//go:build e2ev2 + +package lifecycle + +import ( + "testing" +) + +func TestDeriveClusterName(t *testing.T) { + tests := []struct { + name string + jobID string + variant string + otherJob string + otherVar string + wantSame bool + wantEmpty bool + }{ + { + name: "When the same inputs are provided, it should return the same name", + jobID: "job-123", + variant: "public", + otherJob: "job-123", + otherVar: "public", + wantSame: true, + }, + { + name: "When different job IDs are provided, it should return different names", + jobID: "job-123", + variant: "public", + otherJob: "job-456", + otherVar: "public", + wantSame: false, + }, + { + name: "When different variants are provided, it should return different names", + jobID: "job-123", + variant: "public", + otherJob: "job-123", + otherVar: "upgrade", + wantSame: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := DeriveClusterName(tt.jobID, tt.variant) + if got == "" { + t.Fatal("expected non-empty cluster name") + } + other := DeriveClusterName(tt.otherJob, tt.otherVar) + if tt.wantSame && got != other { + t.Errorf("expected same name, got %q and %q", got, other) + } + if !tt.wantSame && got == other { + t.Errorf("expected different names, both got %q", got) + } + }) + } +} + +func TestFilterClusterSpecs(t *testing.T) { + specs := []ClusterSpec{ + {Variant: "public", OutputFile: "cluster-name-public"}, + {Variant: "private", OutputFile: "cluster-name-private"}, + {Variant: "upgrade", OutputFile: "cluster-name-upgrade"}, + } + + tests := []struct { + name string + variants string + wantCount int + wantVariants []string + }{ + { + name: "When an empty filter is provided, it should return all specs", + variants: "", + wantCount: 3, + }, + { + name: "When a single variant is specified, it should return only that variant", + variants: "public", + wantCount: 1, + wantVariants: []string{"public"}, + }, + { + name: "When multiple variants are specified, it should return all matching variants", + variants: "public,upgrade", + wantCount: 2, + wantVariants: []string{"public", "upgrade"}, + }, + { + name: "When variants have surrounding whitespace, it should trim and match", + variants: " public , upgrade ", + wantCount: 2, + }, + { + name: "When a non-existent variant is specified, it should return no specs", + variants: "nonexistent", + wantCount: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := FilterClusterSpecs(specs, tt.variants) + if len(got) != tt.wantCount { + t.Errorf("expected %d specs, got %d", tt.wantCount, len(got)) + } + if len(tt.wantVariants) > 0 { + have := map[string]bool{} + for _, s := range got { + have[s.Variant] = true + } + for _, v := range tt.wantVariants { + if !have[v] { + t.Errorf("expected variant %q in results", v) + } + } + } + }) + } +} + +func TestFilterTestMatrix(t *testing.T) { + matrix := TestMatrix{ + Parallel: []TestGroup{ + {Name: "public-tests", ClusterFile: "cluster-name-public", LabelFilter: "public"}, + {Name: "private-tests", ClusterFile: "cluster-name-private", LabelFilter: "private"}, + }, + Sequential: []SequentialGroup{ + { + Name: "upgrade-and-chaos", + Steps: []TestGroup{ + {Name: "upgrade", ClusterFile: "cluster-name-upgrade", LabelFilter: "upgrade"}, + {Name: "chaos", ClusterFile: "cluster-name-upgrade", LabelFilter: "chaos"}, + }, + }, + }, + } + + tests := []struct { + name string + specs []ClusterSpec + wantParallel int + wantSequential int + wantSteps int + }{ + { + name: "When all cluster files are present, it should keep all groups", + specs: []ClusterSpec{ + {Variant: "public", OutputFile: "cluster-name-public"}, + {Variant: "private", OutputFile: "cluster-name-private"}, + {Variant: "upgrade", OutputFile: "cluster-name-upgrade"}, + }, + wantParallel: 2, + wantSequential: 1, + wantSteps: 2, + }, + { + name: "When a parallel group's cluster file is missing, it should drop that group", + specs: []ClusterSpec{ + {Variant: "public", OutputFile: "cluster-name-public"}, + {Variant: "upgrade", OutputFile: "cluster-name-upgrade"}, + }, + wantParallel: 1, + wantSequential: 1, + wantSteps: 2, + }, + { + name: "When all steps' cluster files are missing, it should drop the sequential group", + specs: []ClusterSpec{ + {Variant: "public", OutputFile: "cluster-name-public"}, + {Variant: "private", OutputFile: "cluster-name-private"}, + }, + wantParallel: 2, + wantSequential: 0, + }, + { + name: "When no specs are provided, it should return an empty matrix", + specs: nil, + wantParallel: 0, + wantSequential: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := FilterTestMatrix(matrix, tt.specs) + if len(got.Parallel) != tt.wantParallel { + t.Errorf("expected %d parallel groups, got %d", tt.wantParallel, len(got.Parallel)) + } + if len(got.Sequential) != tt.wantSequential { + t.Errorf("expected %d sequential groups, got %d", tt.wantSequential, len(got.Sequential)) + } + if tt.wantSteps > 0 && len(got.Sequential) > 0 { + if len(got.Sequential[0].Steps) != tt.wantSteps { + t.Errorf("expected %d steps, got %d", tt.wantSteps, len(got.Sequential[0].Steps)) + } + } + }) + } +}