From b0a5b144ce7e13ff4108f58f23dd06a63fb3650e Mon Sep 17 00:00:00 2001 From: Sachal Malick Date: Sun, 25 Oct 2026 06:03:31 +0000 Subject: [PATCH 1/3] feat(global-router): preserve reserved GPU compute in separate stacks Signed-off-by: Sachal Malick --- examples/global-router/cdk/.gitignore | 4 +- examples/global-router/cdk/GPU_COMPUTE.md | 122 +++++++++ examples/global-router/cdk/README.md | 4 + examples/global-router/cdk/bin/gpu-compute.ts | 64 +++++ examples/global-router/cdk/lib/gpu-compute.ts | 256 ++++++++++++++++++ .../cdk/scripts/gpu_preflight.py | 88 ++++++ .../cdk/test/gpu-compute.test.ts | 155 +++++++++++ 7 files changed, 691 insertions(+), 2 deletions(-) create mode 100644 examples/global-router/cdk/GPU_COMPUTE.md create mode 100644 examples/global-router/cdk/bin/gpu-compute.ts create mode 100644 examples/global-router/cdk/lib/gpu-compute.ts create mode 100644 examples/global-router/cdk/scripts/gpu_preflight.py create mode 100644 examples/global-router/cdk/test/gpu-compute.test.ts diff --git a/examples/global-router/cdk/.gitignore b/examples/global-router/cdk/.gitignore index 139c399cd529..230359ae2143 100644 --- a/examples/global-router/cdk/.gitignore +++ b/examples/global-router/cdk/.gitignore @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -cdk.out/ +cdk.out*/ cdk.context.json dist/ -node_modules/ +node_modules # Operator-specific settings and credentials. config/*.local.json diff --git a/examples/global-router/cdk/GPU_COMPUTE.md b/examples/global-router/cdk/GPU_COMPUTE.md new file mode 100644 index 000000000000..467afa7f16b8 --- /dev/null +++ b/examples/global-router/cdk/GPU_COMPUTE.md @@ -0,0 +1,122 @@ + + +# Reserved GPU compute + +This app adds GPU capacity without updating the CPU Mocker stacks. Deploy each +layer independently: asset storage, stable VPC/EKS foundation (new regions only), +and reserved GPU compute. Application charts belong in separate workload stacks. +Externally owned On-Demand Capacity Reservations are referenced, never created, +modified, extended or cancelled by this app. + +The compute stack creates an EKS managed node group with a custom launch template +that targets one reservation ID and one instance type in its exact AZ. There is no +unreserved fallback. Fixed minimum/desired/maximum counts prevent automatic scale +down. GPU nodes, launch templates and their role have retain policies; stack deletion +protection is enabled. These guards preserve capacity across application redeploys. +Deliberate compute updates, node failure and replacement, or reservation expiry still +require operational planning. Reservation expiry ends capacity assurance; it does +not automatically stop ordinary On-Demand instances or their billing. + +Node groups cannot move between EKS clusters. Preserve the existing cluster name, +VPC and subnet. Review changesets before changing these settings. Updating a launch +template or AMI deliberately replaces nodes; MINIMAL updates terminate one old node +before its replacement so a full reservation needs no surge slot. Retention preserves +old resources on replacement and can leave billable resources requiring manual cleanup. +Never update the EKS-owned Auto Scaling Group directly. + +## Configure + +Use Node.js 20 or newer and install the locked dependencies with `pnpm install --frozen-lockfile`. +AWS CLI credentials must remain valid throughout deployment. This entry point uses +current operator credentials, with an S3-only asset stack; it creates no persistent +CDK deployment or CloudFormation execution roles. Existing CPU stack commands are unchanged. + +Create `config/gpu-capacity.local.json` (ignored by Git), following this shape with +operator-supplied values. All identifiers below are synthetic. `releaseVersion` must +be a pinned EKS 1.35 NVIDIA release, resolved from the regional SSM parameter +`/aws/service/eks/optimized-ami/1.35/amazon-linux-2023/x86_64/nvidia/recommended/release_version`. +Set `hourlyRate` to the current regional Linux On-Demand price; the preflight checks it. + +```json +{ + "regions": [{ + "region": "us-west-2", + "availabilityZone": "us-west-2a", + "reservationId": "cr-00000000000000000", + "instanceType": "g6.4xlarge", + "count": 2, + "hourlyRate": 1.3232, + "releaseVersion": "1.35.9-20261023", + "clusterName": "example-gpu-cluster", + "subnetId": "subnet-00000000000000000" + }] +} +``` + +For a new region, omit `subnetId` initially and add `foundation` with a nonoverlapping +VPC CIDR and exactly two supported AZs, including the reservation AZ: + +```json +"foundation": { + "cidr": "10.44.0.0/16", + "availabilityZones": ["us-west-2a", "us-west-2b"] +} +``` + +The fleet permits two to four nodes per region, at most four regions/16 nodes, and +at most $7/hour/node. Initial supported types are g5/g6/g6e at 2xlarge or 4xlarge. +The cap covers this configuration; check the account for other GPU fleets separately. + +## Deploy + +Run the read-only preflight before provisioning. It checks live reservation ownership, +state, type, AZ and free slots, AWS prices, and existing cluster/subnet compatibility. +For a new VPC also check that its CIDR does not overlap networks you plan to peer. + +```bash +python3 scripts/gpu_preflight.py config/gpu-capacity.local.json +pnpm build +pnpm test +export CDK_DEFAULT_ACCOUNT=$(aws sts get-caller-identity --query Account --output text) +export CDK_DEFAULT_REGION=us-west-2 +pnpm cdk deploy --app 'node --require ts-node/register bin/gpu-compute.ts' \ + -c gpuConfigPath=config/gpu-capacity.local.json -c gpuRegion=us-west-2 -c gpuMode=assets +``` + +For a new region, deploy the foundation next using the same command with +`gpuMode=foundation`. It creates a private EKS API, two private subnets with NAT +egress, and a pinned NVIDIA device plugin through a VPC-local kubectl provider. +Copy its `GpuSubnetId` output into the local configuration. Do not create a foundation +for an existing cluster. Foundation stacks contain no worker nodes or application charts. + +Then deploy `gpuMode=compute`. Repeat the sequence for each configured region. +For an existing cluster install the NVIDIA plugin through its existing private +admin path; EKS GPU AMIs include drivers but not this Kubernetes plugin: + +```bash +helm template nvidia-device-plugin nvidia-device-plugin \ + --repo https://nvidia.github.io/k8s-device-plugin --version 0.17.1 \ + --namespace kube-system --set-string 'nodeSelector.dynamo\.nvidia\.com/purpose=gpu' \ + --set-json 'affinity=null' | kubectl apply -f - +``` + +The explicit selector restricts the plugin to these GPU nodes without depending on +a separate feature-discovery installation. Nodes use encrypted gp3 root disks, +IMDSv2, no SSH key, and SSM diagnostics access. They are initially untainted so new +clusters can schedule CoreDNS and system pods on this fixed capacity. + +## Verify and operate + +Confirm each managed node group is ACTIVE, both nodes are Ready with +`nvidia.com/gpu: 1` allocatable, and EC2 instances report the expected +`CapacityReservationId`. Run `nvidia-smi` through SSM, then a small GPU pod before +placing inference workloads. Keep full operator evidence outside the repository. + +Check reservation expiry dates separately. Reservations charge for unused slots; +instances occupying them are not charged twice. Additional EKS control planes, NAT, +storage and network traffic have separate costs. Cleanup is deliberate: deletion +protection and retention prevent a normal stack destroy from releasing the fleet. +No command here cancels or extends a reservation. diff --git a/examples/global-router/cdk/README.md b/examples/global-router/cdk/README.md index f29252f04532..7f85a073c274 100644 --- a/examples/global-router/cdk/README.md +++ b/examples/global-router/cdk/README.md @@ -752,3 +752,7 @@ pnpm exec cdk destroy -c deployment=west \ The shared CDK bootstrap stack is not removed. Keep it for other CDK deployments, or remove it separately only after confirming that no other stack uses its assets and roles. + +## Reserved GPU capacity + +See [Reserved GPU compute](GPU_COMPUTE.md) for independent compute stacks that target externally owned Capacity Reservations. diff --git a/examples/global-router/cdk/bin/gpu-compute.ts b/examples/global-router/cdk/bin/gpu-compute.ts new file mode 100644 index 000000000000..22199a54348e --- /dev/null +++ b/examples/global-router/cdk/bin/gpu-compute.ts @@ -0,0 +1,64 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { readFileSync } from "node:fs"; +import { + App, + LegacyStackSynthesizer, + CliCredentialsStackSynthesizer, +} from "aws-cdk-lib"; +import { + GpuAssetsStack, + GpuFoundationStack, + GpuRegion, + ReservedGpuComputeStack, + validateGpuFleet, +} from "../lib/gpu-compute"; + +const app = new App(); +const file = app.node.tryGetContext("gpuConfigPath"); +if (typeof file !== "string" || !file.endsWith(".local.json")) + throw new Error( + "Select an ignored operator file with -c gpuConfigPath=config/gpu-capacity.local.json", + ); +const regions: GpuRegion[] = JSON.parse(readFileSync(file, "utf8")).regions; +validateGpuFleet(regions); +const selected = regions.find( + (r) => r.region === app.node.tryGetContext("gpuRegion"), +); +if (!selected) throw new Error("Choose a configured gpuRegion"); +const mode = app.node.tryGetContext("gpuMode"); +const props = { + config: selected, + env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: selected.region }, + stackName: `${selected.clusterName}-gpu-${mode}`, + synthesizer: + mode === "assets" + ? new LegacyStackSynthesizer() + : new CliCredentialsStackSynthesizer({ + fileAssetsBucketName: `global-router-cdk-assets-${process.env.CDK_DEFAULT_ACCOUNT}-${selected.region}`, + }), +}; +if (mode === "assets") new GpuAssetsStack(app, "GpuAssets", props); +else if (mode === "foundation") + new GpuFoundationStack(app, "GpuFoundation", props); +else if (mode === "compute") + new ReservedGpuComputeStack(app, "GpuCompute", props); +else + throw new Error( + "Choose gpuMode=assets, gpuMode=foundation or gpuMode=compute", + ); diff --git a/examples/global-router/cdk/lib/gpu-compute.ts b/examples/global-router/cdk/lib/gpu-compute.ts new file mode 100644 index 000000000000..ef965c9675fa --- /dev/null +++ b/examples/global-router/cdk/lib/gpu-compute.ts @@ -0,0 +1,256 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + CfnOutput, + CfnResource, + RemovalPolicy, + Stack, + StackProps, + Tags, + aws_ec2 as ec2, + aws_iam as iam, + aws_s3 as s3, +} from "aws-cdk-lib"; +import * as eks from "aws-cdk-lib/aws-eks-v2"; +import { CfnNodegroup } from "aws-cdk-lib/aws-eks"; +import { KubectlV35Layer } from "@aws-cdk/lambda-layer-kubectl-v35"; +import { Construct } from "constructs"; + +export interface GpuRegion { + readonly region: string; + readonly availabilityZone: string; + readonly reservationId: string; + readonly instanceType: string; + readonly count: number; + readonly hourlyRate: number; + readonly releaseVersion: string; + readonly clusterName: string; + readonly subnetId?: string; + readonly foundation?: { + readonly cidr: string; + readonly availabilityZones: string[]; + }; +} + +export function validateGpuFleet(regions: GpuRegion[]): void { + if (!Array.isArray(regions) || !regions.length || regions.length > 4) + throw new Error("Specify one to four GPU regions"); + const seen = new Set(); + let total = 0; + for (const r of regions) { + if (!r || seen.has(r.region) || !/^[a-z]{2}-[a-z]+-\d$/.test(r.region)) + throw new Error("Regions must be valid and unique"); + seen.add(r.region); + if ( + !/^cr-[a-f0-9]{17}$/.test(r.reservationId) || + !r.availabilityZone.startsWith(r.region) || + !/^[a-z]{2}-[a-z]+-\d[a-z]$/.test(r.availabilityZone) + ) + throw new Error("Specify reservation ID and matching regional AZ"); + if (!/^(g5|g6|g6e)\.(2|4)xlarge$/.test(r.instanceType)) + throw new Error("Unsupported POC GPU instance type"); + if (!Number.isInteger(r.count) || r.count < 2 || r.count > 4) + throw new Error("GPU count must be two to four per region"); + if (!Number.isFinite(r.hourlyRate) || r.hourlyRate <= 0 || r.hourlyRate > 7) + throw new Error("GPU hourly rate must be positive and at most $7"); + if ( + !/^1\.35\.\d+-\d{8}$/.test(r.releaseVersion) || + !/^[A-Za-z0-9][A-Za-z0-9_-]{0,99}$/.test(r.clusterName) + ) + throw new Error("Specify EKS 1.35 NVIDIA release and cluster name"); + if (r.subnetId && !/^subnet-[a-f0-9]{17}$/.test(r.subnetId)) + throw new Error("Invalid GPU subnet ID"); + if (!r.subnetId && !r.foundation) + throw new Error("Specify an existing subnet or a new foundation"); + if ( + r.foundation && + (r.foundation.availabilityZones.length !== 2 || + new Set(r.foundation.availabilityZones).size !== 2 || + !r.foundation.availabilityZones.includes(r.availabilityZone) || + r.foundation.availabilityZones.some( + (az) => !new RegExp(`^${r.region}[a-z]$`).test(az), + )) + ) + throw new Error( + "Foundation requires two distinct regional AZs including reserved AZ", + ); + total += r.count; + } + if (total > 16) throw new Error("GPU fleet must not exceed 16 nodes"); +} + +/** Asset storage only; deployment uses the current operator session without deployment roles. */ +export class GpuAssetsStack extends Stack { + constructor(scope: Construct, id: string, props: StackProps) { + super(scope, id, { ...props, terminationProtection: true }); + new s3.Bucket(this, "Assets", { + bucketName: `global-router-cdk-assets-${this.account}-${this.region}`, + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + enforceSSL: true, + encryption: s3.BucketEncryption.S3_MANAGED, + versioned: true, + removalPolicy: RemovalPolicy.RETAIN, + }); + } +} + +/** Stable cluster/network lifecycle. Never contains application Helm releases. */ +export class GpuFoundationStack extends Stack { + constructor( + scope: Construct, + id: string, + props: StackProps & { config: GpuRegion }, + ) { + super(scope, id, { ...props, terminationProtection: true }); + const c = props.config; + if (!c.foundation) throw new Error("Foundation settings required"); + const vpc = new ec2.Vpc(this, "Vpc", { + ipAddresses: ec2.IpAddresses.cidr(c.foundation.cidr), + availabilityZones: c.foundation.availabilityZones, + natGateways: 1, + subnetConfiguration: [ + { name: "public", subnetType: ec2.SubnetType.PUBLIC, cidrMask: 24 }, + { + name: "private", + subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS, + cidrMask: 20, + }, + ], + }); + const cluster = new eks.Cluster(this, "Cluster", { + clusterName: c.clusterName, + version: eks.KubernetesVersion.V1_35, + bootstrapClusterCreatorAdminPermissions: false, + defaultCapacity: 0, + defaultCapacityType: eks.DefaultCapacityType.NODEGROUP, + endpointAccess: eks.EndpointAccess.PRIVATE, + vpc, + vpcSubnets: [{ subnets: vpc.privateSubnets }], + kubectlProviderOptions: { + kubectlLayer: new KubectlV35Layer(this, "KubectlLayer"), + privateSubnets: vpc.privateSubnets, + }, + }); + cluster.addHelmChart("NvidiaDevicePlugin", { + chart: "nvidia-device-plugin", + repository: "https://nvidia.github.io/k8s-device-plugin", + version: "0.17.1", + release: "nvidia-device-plugin", + namespace: "kube-system", + values: { + affinity: null, + nodeSelector: { "dynamo.nvidia.com/purpose": "gpu" }, + }, + wait: false, + }); + const subnet = vpc.privateSubnets.find( + (s) => s.availabilityZone === c.availabilityZone, + ); + if (!subnet) throw new Error("No private subnet in reservation AZ"); + new CfnOutput(this, "GpuSubnetId", { value: subnet.subnetId }); + new CfnOutput(this, "ClusterName", { value: cluster.clusterName }); + new CfnOutput(this, "VpcId", { value: vpc.vpcId }); + // Keep cluster dependencies together on removal; a failed first creation may roll back. + for (const child of this.node.findAll()) { + if (child instanceof CfnResource) + child.applyRemovalPolicy(RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE); + } + Tags.of(this).add("Project", "global-router-gpu"); + } +} + +/** Independently deployed GPU nodes targeting externally owned reservations. */ +export class ReservedGpuComputeStack extends Stack { + constructor( + scope: Construct, + id: string, + props: StackProps & { config: GpuRegion }, + ) { + super(scope, id, { ...props, terminationProtection: true }); + const c = props.config; + if (!c.subnetId) + throw new Error( + "Compute requires a preflight-verified private subnet in reservation AZ", + ); + const role = new iam.Role(this, "NodeRole", { + assumedBy: new iam.ServicePrincipal("ec2.amazonaws.com"), + managedPolicies: [ + "AmazonEKSWorkerNodePolicy", + "AmazonEC2ContainerRegistryPullOnly", + "AmazonEKS_CNI_Policy", + "AmazonSSMManagedInstanceCore", + ].map((name) => iam.ManagedPolicy.fromAwsManagedPolicyName(name)), + }); + role.applyRemovalPolicy(RemovalPolicy.RETAIN); + const template = new ec2.CfnLaunchTemplate(this, "GpuLaunchTemplate", { + launchTemplateData: { + instanceType: c.instanceType, + capacityReservationSpecification: { + capacityReservationTarget: { capacityReservationId: c.reservationId }, + }, + metadataOptions: { httpTokens: "required", httpPutResponseHopLimit: 1 }, + blockDeviceMappings: [ + { + deviceName: "/dev/xvda", + ebs: { + encrypted: true, + volumeSize: 200, + volumeType: "gp3", + deleteOnTermination: true, + }, + }, + ], + tagSpecifications: [ + { + resourceType: "instance", + tags: [ + { key: "Project", value: "global-router-gpu" }, + { key: "Name", value: `${c.clusterName}-reserved-gpu` }, + ], + }, + ], + }, + }); + template.applyRemovalPolicy(RemovalPolicy.RETAIN); + const nodes = new CfnNodegroup(this, "GpuNodes", { + clusterName: c.clusterName, + nodegroupName: "global-router-reserved-gpu", + nodeRole: role.roleArn, + subnets: [c.subnetId], + capacityType: "ON_DEMAND", + amiType: "AL2023_x86_64_NVIDIA", + version: "1.35", + releaseVersion: c.releaseVersion, + launchTemplate: { + id: template.ref, + version: template.attrLatestVersionNumber, + }, + scalingConfig: { + minSize: c.count, + desiredSize: c.count, + maxSize: c.count, + }, + updateConfig: { maxUnavailable: 1, updateStrategy: "MINIMAL" }, + labels: { "dynamo.nvidia.com/purpose": "gpu" }, + }); + nodes.applyRemovalPolicy(RemovalPolicy.RETAIN); + new CfnOutput(this, "NodegroupName", { value: nodes.ref }); + new CfnOutput(this, "NodeRoleArn", { value: role.roleArn }); + Tags.of(this).add("Project", "global-router-gpu"); + } +} diff --git a/examples/global-router/cdk/scripts/gpu_preflight.py b/examples/global-router/cdk/scripts/gpu_preflight.py new file mode 100644 index 000000000000..cda1556d7e3d --- /dev/null +++ b/examples/global-router/cdk/scripts/gpu_preflight.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Read-only AWS checks before provisioning reserved GPU compute (AWS CLI required).""" +import argparse +import json +from pathlib import Path +import subprocess + + +def aws(region, service, *args): + return json.loads(subprocess.check_output([ + "aws", service, *args, "--region", region, "--output", "json", "--no-cli-pager" + ], text=True)) + + +def check(config, region_filter=None): + regions = config["regions"] + if not 1 <= len(regions) <= 4 or len({r["region"] for r in regions}) != len(regions): + raise ValueError("Specify one to four unique regions") + if sum(r["count"] for r in regions) > 16: + raise ValueError("Fleet exceeds 16 nodes") + owner = aws("us-east-1", "sts", "get-caller-identity")["Account"] + selected = [r for r in regions if region_filter is None or r["region"] == region_filter] + if not selected: + raise ValueError("Requested region is not configured") + total = 0.0 + for r in selected: + region = r["region"] + count = r["count"] + if not isinstance(count, int) or not 2 <= count <= 4: + raise ValueError("Specify two to four nodes per region") + cr = aws(region, "ec2", "describe-capacity-reservations", "--capacity-reservation-ids", r["reservationId"])["CapacityReservations"][0] + expected = {"State": "active", "OwnerId": owner, "InstanceType": r["instanceType"], + "AvailabilityZone": r["availabilityZone"], "InstancePlatform": "Linux/UNIX", "Tenancy": "default"} + for key, value in expected.items(): + if cr.get(key) != value: + raise ValueError(f"{region}: reservation {key} does not match") + if cr["TotalInstanceCount"] < count: + raise ValueError(f"{region}: reservation too small") + existing = aws(region, "ec2", "describe-instances", "--filters", + f"Name=capacity-reservation-id,Values={r['reservationId']}", + "Name=instance-state-name,Values=pending,running")["Reservations"] + owned = [i for group in existing for i in group["Instances"] + if {t["Key"]: t["Value"] for t in i.get("Tags", [])}.get("Project") == "global-router-gpu"] + if cr["AvailableInstanceCount"] + len(owned) < count: + raise ValueError(f"{region}: slots occupied by other compute") + filters = [{"Type": "TERM_MATCH", "Field": k, "Value": v} for k, v in { + "regionCode": region, "instanceType": r["instanceType"], "operatingSystem": "Linux", + "tenancy": "Shared", "preInstalledSw": "NA", "capacitystatus": "Used"}.items()] + prices = aws("us-east-1", "pricing", "get-products", "--service-code", "AmazonEC2", "--filters", json.dumps(filters))["PriceList"] + rates = [float(d["pricePerUnit"]["USD"]) for p in prices + for term in json.loads(p)["terms"].get("OnDemand", {}).values() + for d in term["priceDimensions"].values() if "On Demand Linux" in d.get("description", "")] + if len(set(rates)) != 1 or not 0 < rates[0] <= 7: + raise ValueError(f"{region}: price missing, ambiguous, or above $7/hour") + if abs(rates[0] - r["hourlyRate"]) > 0.00001: + raise ValueError(f"{region}: configured price differs from AWS; refresh hourlyRate") + if r.get("subnetId"): + subnet = aws(region, "ec2", "describe-subnets", "--subnet-ids", r["subnetId"])["Subnets"][0] + cluster = aws(region, "eks", "describe-cluster", "--name", r["clusterName"])["cluster"] + if (subnet["AvailabilityZone"] != r["availabilityZone"] or subnet["MapPublicIpOnLaunch"] or + subnet["VpcId"] != cluster["resourcesVpcConfig"]["vpcId"] or subnet["AvailableIpAddressCount"] < count * 10 or + cluster["status"] != "ACTIVE" or cluster["version"] != "1.35"): + raise ValueError(f"{region}: subnet or cluster is incompatible") + total += count * rates[0] + print(f"{region}: {count} x {r['instanceType']}, reservation verified, ${count * rates[0]:.5f}/hour; expires {cr.get('EndDate', 'no fixed end')}") + print(f"Selected compute total: ${total:.5f}/hour (excludes EBS/network/EKS/NAT)") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("config", type=Path) + parser.add_argument("--region") + args = parser.parse_args() + check(json.loads(args.config.read_text()), args.region) diff --git a/examples/global-router/cdk/test/gpu-compute.test.ts b/examples/global-router/cdk/test/gpu-compute.test.ts new file mode 100644 index 000000000000..ffc6b8a7e5d0 --- /dev/null +++ b/examples/global-router/cdk/test/gpu-compute.test.ts @@ -0,0 +1,155 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { App } from "aws-cdk-lib"; +import { Template, Match } from "aws-cdk-lib/assertions"; +import { + GpuAssetsStack, + ReservedGpuComputeStack, + GpuFoundationStack, + GpuRegion, + validateGpuFleet, +} from "../lib/gpu-compute"; + +const config: GpuRegion = { + region: "us-west-2", + availabilityZone: "us-west-2a", + reservationId: "cr-00000000000000000", + instanceType: "g6.4xlarge", + count: 2, + hourlyRate: 1.4, + releaseVersion: "1.35.9-20261023", + clusterName: "example-gpu", + subnetId: "subnet-00000000000000000", +}; +test("compute targets only the supplied reservation and cannot scale down or surge", () => { + const stack = new ReservedGpuComputeStack(new App(), "Compute", { config }); + const t = Template.fromStack(stack); + t.resourceCountIs("AWS::EKS::Cluster", 0); + t.resourceCountIs("AWS::EC2::CapacityReservation", 0); + t.resourceCountIs("AWS::EC2::VPC", 0); + t.hasResourceProperties("AWS::EC2::LaunchTemplate", { + LaunchTemplateData: { + InstanceType: config.instanceType, + CapacityReservationSpecification: { + CapacityReservationTarget: { + CapacityReservationId: config.reservationId, + }, + }, + MetadataOptions: { HttpTokens: "required", HttpPutResponseHopLimit: 1 }, + ImageId: Match.absent(), + UserData: Match.absent(), + }, + }); + t.hasResourceProperties("AWS::EKS::Nodegroup", { + ClusterName: config.clusterName, + Subnets: [config.subnetId], + AmiType: "AL2023_x86_64_NVIDIA", + ReleaseVersion: config.releaseVersion, + ScalingConfig: { MinSize: 2, DesiredSize: 2, MaxSize: 2 }, + UpdateConfig: { MaxUnavailable: 1, UpdateStrategy: "MINIMAL" }, + }); + for (const type of [ + "AWS::EKS::Nodegroup", + "AWS::EC2::LaunchTemplate", + "AWS::IAM::Role", + ]) { + t.hasResource(type, { + DeletionPolicy: "Retain", + UpdateReplacePolicy: "Retain", + }); + } + assert.equal(stack.terminationProtection, true); +}); +test("foundation supplies private control plane and no GPU lifecycle resources", () => { + const stack = new GpuFoundationStack(new App(), "Foundation", { + env: { region: "us-west-2", account: "111111111111" }, + config: { + ...config, + subnetId: undefined, + foundation: { + cidr: "10.44.0.0/16", + availabilityZones: ["us-west-2a", "us-west-2b"], + }, + }, + }); + const t = Template.fromStack(stack); + t.resourceCountIs("AWS::EKS::Nodegroup", 0); + t.resourceCountIs("AWS::EC2::CapacityReservation", 0); + t.hasResourceProperties("AWS::EKS::Cluster", { + ResourcesVpcConfig: { + EndpointPrivateAccess: true, + EndpointPublicAccess: false, + }, + }); + for (const type of [ + "AWS::EKS::Cluster", + "AWS::EC2::VPC", + "AWS::EC2::Subnet", + "AWS::IAM::Role", + ]) { + t.hasResource(type, { + DeletionPolicy: "RetainExceptOnCreate", + UpdateReplacePolicy: "Retain", + }); + } + assert.equal(stack.terminationProtection, true); +}); +test("asset bootstrap has no persistent deployment roles", () => { + const stack = new GpuAssetsStack(new App(), "Assets", { + env: { region: "us-west-2", account: "111111111111" }, + }); + const t = Template.fromStack(stack); + t.resourceCountIs("AWS::IAM::Role", 0); + t.resourceCountIs("AWS::EKS::Cluster", 0); + t.hasResourceProperties("AWS::S3::Bucket", { + PublicAccessBlockConfiguration: { + BlockPublicAcls: true, + BlockPublicPolicy: true, + IgnorePublicAcls: true, + RestrictPublicBuckets: true, + }, + }); +}); +test("fleet rejects duplicate regions, counts, cost violations and incompatible types", () => { + validateGpuFleet([config]); + for (const value of [ + { count: 1 }, + { count: 5 }, + { count: 2.5 }, + { hourlyRate: 7.01 }, + { hourlyRate: 0 }, + { instanceType: "p5.48xlarge" }, + { reservationId: "" }, + { availabilityZone: "us-east-2a" }, + { subnetId: "invalid" }, + { releaseVersion: "latest" }, + { subnetId: undefined }, + { + foundation: { + cidr: "10.44.0.0/16", + availabilityZones: ["us-west-2a", "us-west-2a"], + }, + }, + ]) { + assert.throws(() => validateGpuFleet([{ ...config, ...value }])); + } + assert.throws(() => validateGpuFleet([config, config]), /unique/); + assert.throws(() => validateGpuFleet([])); +}); From 0997e8b314708bafe2529cf91eecba30977de665 Mon Sep 17 00:00:00 2001 From: Sachal Date: Sun, 25 Oct 2026 00:35:20 -0700 Subject: [PATCH 2/3] feat(global-router): integrate Karmada and MCS support (#17270) Signed-off-by: Anna Tchernych Co-authored-by: Anna Tchernych Co-authored-by: Claude Opus 5.5 (1M context) --- examples/global-router/cdk/README.md | 152 +++++++- .../cdk/bin/dynamo-mocker-cpu.ts | 25 +- examples/global-router/cdk/crds/README.md | 58 +++ .../multicluster.x-k8s.io_serviceexports.json | 119 +++++++ .../multicluster.x-k8s.io_serviceimports.json | 171 +++++++++ examples/global-router/cdk/lib/cidr.ts | 82 +++++ examples/global-router/cdk/lib/config.ts | 31 ++ examples/global-router/cdk/lib/const.ts | 23 +- .../cdk/lib/dynamo-mocker-cpu-stack.ts | 79 ++--- .../cdk/lib/inter-region-peering.ts | 42 ++- .../global-router/cdk/lib/multi-cluster.ts | 326 +++++++++++++++++ .../cdk/test/inter-region-peering.test.ts | 71 +++- .../cdk/test/multi-cluster.test.ts | 329 ++++++++++++++++++ 13 files changed, 1447 insertions(+), 61 deletions(-) create mode 100644 examples/global-router/cdk/crds/README.md create mode 100644 examples/global-router/cdk/crds/multicluster.x-k8s.io_serviceexports.json create mode 100644 examples/global-router/cdk/crds/multicluster.x-k8s.io_serviceimports.json create mode 100644 examples/global-router/cdk/lib/cidr.ts create mode 100644 examples/global-router/cdk/lib/multi-cluster.ts create mode 100644 examples/global-router/cdk/test/multi-cluster.test.ts diff --git a/examples/global-router/cdk/README.md b/examples/global-router/cdk/README.md index 7f85a073c274..1b8ffb8570e9 100644 --- a/examples/global-router/cdk/README.md +++ b/examples/global-router/cdk/README.md @@ -97,7 +97,7 @@ Global Router. The example places the Global Router in Ohio. Ohio and West use non-overlapping VPC ranges `10.42.0.0/16` and `10.43.0.0/16`. Deploy the regional stacks separately with `deployment=ohio` and `deployment=west`. The `-c deployment=both` mode then creates only a single -inter-Region VPC peering connection and private routes in both directions. The peering connection +inter-Region VPC peering connection and routes in both directions. The peering connection and routes do not create public ingress. Public regional ingress addresses are client entry points; the Global Router uses a separate private endpoint for each DGD Frontend. @@ -265,6 +265,142 @@ instead, with curl directed to local `127.0.0.1:8080`. > still exposes the SSH daemon to the public internet. Rotate keys promptly and destroy the stack > when the environment is not in use. +## Multi-cluster: Karmada and Multi-Cluster Services (MCS) + +A deployment with `multiCluster` set in `lib/const.ts` is ready for Karmada and the Multi-Cluster +Services (MCS) API from SIG Multicluster. Ohio is the Karmada hub (`aws-ohio`) and West is a +member (`aws-west`). The member names match the Global Router's `site_id` values. For every such +cluster, CDK: + +- adds an ingress rule for all TCP from each peer VPC range to the EKS cluster security group. + MCS EndpointSlices list pod addresses, so a caller in the other cluster, such as a Global Router + dialing a Relay it found through discovery, connects to the pod directly; without this rule only + the Relay's NLB is reachable. Pods use their node's security groups, and the nodes get the + cluster security group because their launch templates set none. A second rule admits the ICMP + "fragmentation needed" message from the peer: inter-Region peering carries smaller packets than + the pods' MTU, and Path MTU Discovery needs that message to find the limit; +- excludes the peer VPC ranges from source NAT in the VPC CNI (`aws-node` + `AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS`), so traffic between the clusters keeps its pod address; +- installs the MCS `ServiceExport` and `ServiceImport` CRDs (`crds/`, from Karmada 1.19.0). CDK + keeps them if they are removed from the stack or a deployment rolls back, because they may + predate the stack and deleting a CRD deletes every `ServiceExport` and `ServiceImport`; +- in the hub only, and only with `-c karmada=true`, installs the Karmada 1.19.0 control plane + from the release Helm chart into `karmada-system`, with every image pinned and the + `PropagateDeps` feature gate on (the chart turns it off, and the import step below needs it). + Its API Service is an internal NLB on TCP 5443 that only the configured member VPC ranges can + reach, and its certificate covers `*.elb..amazonaws.com`. + +Karmada's etcd keeps its data on the node's own disk (`hostPath`) and is not pinned to a node. +If the etcd pod moves, for example when the node group replaces a node, the hub loses its state +and does not recover by itself: install the control plane again and register the members again. + +Deploy the hub first, then the members, then `deployment=both`. Leave out `-c karmada=true` until +the running clusters are moved off their hand-installed Karmada (see the last section): + +```bash +pnpm exec cdk deploy -c deployment=ohio -c karmada=true # add the routerImage flags if the Router is deployed +pnpm exec cdk deploy -c deployment=west +pnpm exec cdk deploy --all -c deployment=both +``` + +Pass `-c karmada=true` on every later Ohio deployment. Without it, CDK stops managing the control +plane but leaves it installed: CDK never uninstalls the chart, because its uninstall hook also +deletes the MCS CRDs and the `karmada-system` namespace. + +### Let the bastions reach the other VPC + +Each bastion sits in a public subnet. `publicRouteTableIds` in `POC_PEERING_OHIO` and +`POC_PEERING_WEST` lists the public subnets' route tables, so `deployment=both` routes them over +the peering as well, and each bastion can reach the other VPC, including the Karmada API load +balancer. The private routes keep their logical IDs. If a regional VPC is rebuilt, find its +public route tables again (the ones with an internet gateway route): + +```bash +aws ec2 describe-route-tables --region us-east-2 \ + --filters Name=vpc-id,Values= Name=route.gateway-id,Values='igw-*' \ + --query 'RouteTables[].RouteTableId' --output text +``` + +### Join the clusters to Karmada (Pull mode) + +Each member's `karmada-agent` dials the hub, so the hub never needs a member's private EKS API. +The join uses a short-lived token from the hub, so it is a step after deployment, not a +CloudFormation resource. Install `karmadactl` 1.19.0 on each bastion first: + +```bash +curl -fLO https://github.com/karmada-io/karmada/releases/download/v1.19.0/karmadactl-linux-amd64.tgz +tar xzf karmadactl-linux-amd64.tgz && sudo install -m 0755 karmadactl /usr/local/bin/ +``` + +On the Ohio bastion, write the Karmada kubeconfig, point it at the load balancer, and create a +join token. The `KarmadaKubeconfigCommand` and `KarmadaApiEndpointCommand` stack outputs print +the first two commands. The deployment does not wait for the load balancer's address, so if +`echo $LB` prints nothing, wait a minute and run the `LB=` line again: + +```bash +kubectl get secret -n karmada-system karmada-kubeconfig -o jsonpath='{.data.kubeconfig}' | base64 -d > karmada-apiserver.config +LB=$(kubectl get service -n karmada-system karmada-apiserver -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') +echo $LB +kubectl config set-cluster karmada-apiserver --server "https://$LB:5443" --kubeconfig karmada-apiserver.config +karmadactl token create --print-register-command --kubeconfig karmada-apiserver.config +``` + +`token create` prints a `karmadactl register` command that already holds the load balancer +address, the token, and the CA hash. Run that command on each cluster's bastion, adding +`--cluster-name` with the cluster's `KarmadaMemberName` output. On Ohio, also add +`--namespace karmada-agent`. The agent's default namespace is `karmada-system`, where the control +plane already has the `karmada-kubeconfig` Secret, so `register` would stop without registering: + +```bash +karmadactl register --cluster-name aws-west +karmadactl register --cluster-name aws-ohio --namespace karmada-agent +``` + +To remove Ohio later, pass the same `--namespace karmada-agent` to `karmadactl unregister`. +With the default namespace, it would delete the Secret that the control plane uses. + +Then, on the Ohio bastion, both members should be `Pull` and `Ready`: + +```bash +kubectl --kubeconfig karmada-apiserver.config get clusters +``` + +### Export and import Services + +Create a Service and its `ServiceExport` directly in the cluster that runs the pods, not +through Karmada propagation. Kubernetes copies a Service's labels onto its EndpointSlices, so a +Service that Karmada propagates gives its slices Karmada's labels. In a test with Karmada 1.19.0, +deleting the `ServiceExport` of such a Service also made `karmada-agent` delete the Service's +EndpointSlice, and exporting it again found no slice until the Service changed. + +Karmada collects the EndpointSlices of every exported Service on its control plane as +`imported--`. To use a Service from a cluster, create a `ServiceImport` on the +Karmada control plane and propagate it there with a `PropagationPolicy`: that cluster gets a +`derived-` Service whose EndpointSlices list the pods of every exporting cluster, its +own included. Karmada delivers them as the `ServiceImport`'s dependencies. Its webhook turns on +`propagateDeps` in any policy that selects a `ServiceImport`, so the policy need not set it, but +the controller's `PropagateDeps` feature gate must be on, which CDK does. + +### Moving the current clusters to the CDK-managed Karmada + +The running clusters have a Karmada control plane that was installed by hand with +`karmadactl init`, and Helm cannot install over it. Deployments without `-c karmada=true` are +safe: they add the network settings and take over the MCS CRDs that Karmada propagated. Before +deploying Ohio with `-c karmada=true`: + +1. Deploy `deployment=both` to add the public routes (above), so both bastions reach the Karmada + API. +2. On each bastion, remove that cluster's agent. `karmadactl` reads the hub address from the + agent's own kubeconfig: `karmadactl unregister aws-west --cluster-kubeconfig ~/.kube/config` + on West, and the same with `aws-ohio` on Ohio. +3. On the Ohio bastion, remove the control plane: `karmadactl deinit`, then + `kubectl delete namespace karmada-system`. +4. In both clusters, delete what the old Karmada created: the `mcs-test` namespace, the + `derived-*` Services and their `imported-*` EndpointSlices, and in West the + Karmada-propagated `relay-mocker-west-1` Service and its `ServiceExport`. + +Then deploy as above and join the members again. + ## Prerequisites - Node.js 20 or later @@ -349,10 +485,11 @@ pnpm exec cdk deploy --all -c deployment=both \ -c deploymentConfigPath=config/deployments.local.json ``` -One stack owns the peering connection and Ohio private routes; the other owns West private routes. -Each listed private route table receives a native CloudFormation route to the opposite VPC CIDR. -Public route tables are not changed. The regional load balancer controller applies ingress from -NLB Service source ranges. The private NLB DNS name resolves to its private node addresses; +One stack owns the peering connection and Ohio routes; the other owns West routes. +Each listed private or public route table receives a native CloudFormation route to the opposite VPC CIDR. +Populate `publicRouteTableIds` to enable private peer access from bastions. +The regional load balancer controller applies ingress from NLB Service source ranges. +The `multiCluster` configuration additionally admits peer pod traffic through the cluster security group. The private NLB DNS name resolves to its private node addresses; peering DNS-resolution options for EC2 public hostnames are not needed for that name. Verify DNS resolution and connectivity from the Router's private network after peering becomes active. @@ -713,7 +850,7 @@ See: - [Amazon EC2 key pairs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html) - [Amazon EKS VPC and subnet requirements](https://docs.aws.amazon.com/eks/latest/userguide/network-reqs.html) - [EFA limitations](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa.html) -- [Dynamo topology-aware KV transfer](https://github.com/ai-dynamo/dynamo/blob/main/docs/fern/pages/developer-guide/knowledge-base/kubernetes/multinode/topology-aware-kv-transfer.md) +- [Dynamo topology-aware KV transfer](https://github.com/ai-dynamo/dynamo/blob/main/docs/fern/pages/developer-guide/knowledge-base/modular-components/router/topology-aware-kv-transfer.md) - [Dynamo Mocker simulation](https://github.com/ai-dynamo/dynamo/blob/main/docs/fern/pages/kubernetes/operations/simulation-with-dynosim/mocker-live-simulation.mdx) ## Main Configuration @@ -740,6 +877,9 @@ See: | `MOCKER_OHIO_1_DGD.mockerSpeedupRatio` | `1` | Simulated worker speed multiplier | | `US_EAST2_DEPLOYMENT.dgdPools` | `MOCKER_OHIO_1_POOL` | Infrastructure pools created by this deployment | | `US_EAST2_DEPLOYMENT.dgds` | `MOCKER_OHIO_1_DGD` | DGD workloads created by this deployment | +| `US_EAST2_DEPLOYMENT.multiCluster.memberName` | `aws-ohio` | Karmada member name and Router site ID | +| `US_EAST2_DEPLOYMENT.multiCluster.peerCidrs` | `10.43.0.0/16` | Peer VPCs whose pods may reach this cluster's pods | +| `US_EAST2_DEPLOYMENT.multiCluster.karmadaControlPlane.version` | `1.19.0` | Karmada release installed in the hub with `-c karmada=true`; must equal `KARMADA_VERSION` | ## Destroy diff --git a/examples/global-router/cdk/bin/dynamo-mocker-cpu.ts b/examples/global-router/cdk/bin/dynamo-mocker-cpu.ts index abf53d0f532f..776e05c52e19 100644 --- a/examples/global-router/cdk/bin/dynamo-mocker-cpu.ts +++ b/examples/global-router/cdk/bin/dynamo-mocker-cpu.ts @@ -23,6 +23,7 @@ import type { DeploymentConfig } from "../lib/config"; import { loadGlobalRouterDeployment } from "../lib/global-router-deployment"; import { DynamoMockerCpuStack } from "../lib/dynamo-mocker-cpu-stack"; import { addInterRegionPeering } from "../lib/inter-region-peering"; +import { withKarmadaControlPlane } from "../lib/multi-cluster"; const app = new App(); const deploymentName = app.node.tryGetContext("deployment") ?? "ohio"; @@ -31,12 +32,16 @@ const globalRouter = loadGlobalRouterDeployment( app.node.tryGetContext("routerImage"), app.node.tryGetContext("routerConfigPath"), ); +const installKarmada = contextFlag("karmada"); if (deploymentName === "both") { if (globalRouter !== undefined) { throw new Error("Deploy the Global Router with -c deployment=ohio after peering."); } + if (installKarmada) { + throw new Error("Install Karmada with -c deployment=ohio."); + } // Existing regional stacks are deployed separately. This mode changes only - // the connection and private routes between their VPCs. + // the connection and routes between their VPCs. addInterRegionPeering( app, settings.peering.ohio, @@ -47,7 +52,23 @@ if (deploymentName === "both") { if (globalRouter !== undefined && deploymentName !== "ohio") { throw new Error("The Global Router POC workload is only supported in Ohio."); } - createStack("DynamoMockerCpu", selectDeployment(deploymentName, settings.deployments), globalRouter); + createStack( + "DynamoMockerCpu", + withKarmadaControlPlane(selectDeployment(deploymentName, settings.deployments), installKarmada), + globalRouter, + ); +} + +/** `-c =true` or `false`; unset means false. */ +function contextFlag(name: string): boolean { + const value: unknown = app.node.tryGetContext(name); + if (value === undefined || value === false || value === "false") { + return false; + } + if (value === true || value === "true") { + return true; + } + throw new Error(`-c ${name} must be true or false.`); } function createStack( diff --git a/examples/global-router/cdk/crds/README.md b/examples/global-router/cdk/crds/README.md new file mode 100644 index 000000000000..aa584d178254 --- /dev/null +++ b/examples/global-router/cdk/crds/README.md @@ -0,0 +1,58 @@ + + +# SIG Multicluster CRDs + +`ServiceExport` and `ServiceImport` (`multicluster.x-k8s.io/v1alpha1`), installed in every +cluster that sets `multiCluster`. They are copied unchanged from Karmada 1.19.0, +`charts/karmada/_crds/bases/multicluster/`, and stored as JSON so the CDK app needs no YAML +parser. + +The upstream files carry this notice, which JSON cannot hold as a comment: + +```text +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` + +To refresh them for another Karmada release `X.Y.Z`, run this from the CDK app's directory. A +file is replaced only when its download converts to a CRD; otherwise the loop prints +`Not refreshed`: + +```bash +for kind in serviceexports serviceimports; do + file="multicluster.x-k8s.io_${kind}" + curl -fsSL -o "/tmp/${file}.yaml" "https://raw.githubusercontent.com/karmada-io/karmada/vX.Y.Z/charts/karmada/_crds/bases/multicluster/${file}.yaml" && + python3 -c 'import json, sys, yaml; crd = yaml.safe_load(open(sys.argv[1])); assert crd["kind"] == "CustomResourceDefinition"; print(json.dumps(crd, indent=2))' "/tmp/${file}.yaml" > "/tmp/${file}.json" && + mv "/tmp/${file}.json" "crds/${file}.json" || + echo "Not refreshed: ${file}" +done +``` + +Then set `KARMADA_VERSION` in `lib/multi-cluster.ts` to the same release, check the cfssl and +kubectl image pins next to it, and update the release in this file. diff --git a/examples/global-router/cdk/crds/multicluster.x-k8s.io_serviceexports.json b/examples/global-router/cdk/crds/multicluster.x-k8s.io_serviceexports.json new file mode 100644 index 000000000000..e7f49782587e --- /dev/null +++ b/examples/global-router/cdk/crds/multicluster.x-k8s.io_serviceexports.json @@ -0,0 +1,119 @@ +{ + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": { + "name": "serviceexports.multicluster.x-k8s.io" + }, + "spec": { + "group": "multicluster.x-k8s.io", + "scope": "Namespaced", + "names": { + "plural": "serviceexports", + "singular": "serviceexport", + "kind": "ServiceExport", + "shortNames": [ + "svcex" + ] + }, + "versions": [ + { + "name": "v1alpha1", + "served": true, + "storage": true, + "subresources": { + "status": {} + }, + "additionalPrinterColumns": [ + { + "name": "Age", + "type": "date", + "jsonPath": ".metadata.creationTimestamp" + } + ], + "schema": { + "openAPIV3Schema": { + "description": "ServiceExport declares that the Service with the same name and namespace as this export should be consumable from other clusters.", + "type": "object", + "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + "type": "string" + }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + "type": "string" + }, + "metadata": { + "type": "object" + }, + "status": { + "description": "status describes the current state of an exported service. Service configuration comes from the Service that had the same name and namespace as this ServiceExport. Populated by the multi-cluster service implementation's controller.", + "type": "object", + "properties": { + "conditions": { + "type": "array", + "items": { + "description": "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }", + "type": "object", + "required": [ + "lastTransitionTime", + "message", + "reason", + "status", + "type" + ], + "properties": { + "lastTransitionTime": { + "description": "lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.", + "type": "string", + "format": "date-time" + }, + "message": { + "description": "message is a human readable message indicating details about the transition. This may be an empty string.", + "type": "string", + "maxLength": 32768 + }, + "observedGeneration": { + "description": "observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.", + "type": "integer", + "format": "int64", + "minimum": 0 + }, + "reason": { + "description": "reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.", + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$" + }, + "status": { + "description": "status of the condition, one of True, False, Unknown.", + "type": "string", + "enum": [ + "True", + "False", + "Unknown" + ] + }, + "type": { + "description": "type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)", + "type": "string", + "maxLength": 316, + "pattern": "^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$" + } + } + }, + "x-kubernetes-list-map-keys": [ + "type" + ], + "x-kubernetes-list-type": "map" + } + } + } + } + } + } + } + ] + } +} diff --git a/examples/global-router/cdk/crds/multicluster.x-k8s.io_serviceimports.json b/examples/global-router/cdk/crds/multicluster.x-k8s.io_serviceimports.json new file mode 100644 index 000000000000..c6a8dcd2c0bf --- /dev/null +++ b/examples/global-router/cdk/crds/multicluster.x-k8s.io_serviceimports.json @@ -0,0 +1,171 @@ +{ + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": { + "name": "serviceimports.multicluster.x-k8s.io" + }, + "spec": { + "group": "multicluster.x-k8s.io", + "scope": "Namespaced", + "names": { + "plural": "serviceimports", + "singular": "serviceimport", + "kind": "ServiceImport", + "shortNames": [ + "svcim" + ] + }, + "versions": [ + { + "name": "v1alpha1", + "served": true, + "storage": true, + "subresources": { + "status": {} + }, + "additionalPrinterColumns": [ + { + "name": "Type", + "type": "string", + "description": "The type of this ServiceImport", + "jsonPath": ".spec.type" + }, + { + "name": "IP", + "type": "string", + "description": "The VIP for this ServiceImport", + "jsonPath": ".spec.ips" + }, + { + "name": "Age", + "type": "date", + "jsonPath": ".metadata.creationTimestamp" + } + ], + "schema": { + "openAPIV3Schema": { + "description": "ServiceImport describes a service imported from clusters in a ClusterSet.", + "type": "object", + "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + "type": "string" + }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + "type": "string" + }, + "metadata": { + "type": "object" + }, + "spec": { + "description": "spec defines the behavior of a ServiceImport.", + "type": "object", + "required": [ + "ports", + "type" + ], + "properties": { + "ips": { + "description": "ip will be used as the VIP for this service when type is ClusterSetIP.", + "type": "array", + "maxItems": 1, + "items": { + "type": "string" + } + }, + "ports": { + "type": "array", + "items": { + "description": "ServicePort represents the port on which the service is exposed", + "type": "object", + "required": [ + "port" + ], + "properties": { + "appProtocol": { + "description": "The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and http://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol. Field can be enabled with ServiceAppProtocol feature gate.", + "type": "string" + }, + "name": { + "description": "The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service.", + "type": "string" + }, + "port": { + "description": "The port that will be exposed by this service.", + "type": "integer", + "format": "int32" + }, + "protocol": { + "description": "The IP protocol for this port. Supports \"TCP\", \"UDP\", and \"SCTP\". Default is TCP.", + "type": "string" + } + } + }, + "x-kubernetes-list-type": "atomic" + }, + "sessionAffinity": { + "description": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. Ignored when type is Headless More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", + "type": "string" + }, + "sessionAffinityConfig": { + "description": "sessionAffinityConfig contains session affinity configuration.", + "type": "object", + "properties": { + "clientIP": { + "description": "clientIP contains the configurations of Client IP based session affinity.", + "type": "object", + "properties": { + "timeoutSeconds": { + "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be >0 && <=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours).", + "type": "integer", + "format": "int32" + } + } + } + } + }, + "type": { + "description": "type defines the type of this service. Must be ClusterSetIP or Headless.", + "type": "string", + "enum": [ + "ClusterSetIP", + "Headless" + ] + } + } + }, + "status": { + "description": "status contains information about the exported services that form the multi-cluster service referenced by this ServiceImport.", + "type": "object", + "properties": { + "clusters": { + "description": "clusters is the list of exporting clusters from which this service was derived.", + "type": "array", + "items": { + "description": "ClusterStatus contains service configuration mapped to a specific source cluster", + "type": "object", + "required": [ + "cluster" + ], + "properties": { + "cluster": { + "description": "cluster is the name of the exporting cluster. Must be a valid RFC-1123 DNS label.", + "type": "string" + } + } + }, + "x-kubernetes-list-map-keys": [ + "cluster" + ], + "x-kubernetes-list-type": "map" + } + } + } + } + } + } + } + ] + } +} diff --git a/examples/global-router/cdk/lib/cidr.ts b/examples/global-router/cdk/lib/cidr.ts new file mode 100644 index 000000000000..64ac2d904a9d --- /dev/null +++ b/examples/global-router/cdk/lib/cidr.ts @@ -0,0 +1,82 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +interface Ipv4Range { + readonly octets: readonly number[]; + readonly prefix: number; + /** The written address as a number. */ + readonly address: number; +} + +function parseIpv4Cidr(value: string): Ipv4Range | undefined { + const match = + /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\/(\d{1,2})$/.exec( + value, + ); + if (match === null) { + return undefined; + } + const octets = match.slice(1, 5).map(Number); + const prefix = Number(match[5]); + if (octets.some((octet) => octet > 255) || prefix > 32) { + return undefined; + } + const address = octets.reduce((sum, octet) => sum * 256 + octet, 0); + return { octets, prefix, address }; +} + +/** True for an IPv4 range such as `10.42.0.0/16`. */ +export function isValidIpv4Cidr(value: string): boolean { + return parseIpv4Cidr(value) !== undefined; +} + +/** + * True for a private, network-aligned range from /16 to /24 that is written + * without leading zeros, such as `10.42.0.0/16`. + */ +export function isValidVpcCidr(value: string): boolean { + const range = parseIpv4Cidr(value); + if (range === undefined || range.prefix < 16 || range.prefix > 24) { + return false; + } + const [first, second] = range.octets; + const isPrivate = + first === 10 || + (first === 172 && second >= 16 && second <= 31) || + (first === 192 && second === 168); + return ( + isPrivate && + range.address % 2 ** (32 - range.prefix) === 0 && + `${range.octets.join(".")}/${range.prefix}` === value + ); +} + +/** True when two IPv4 ranges share an address. */ +export function cidrsOverlap(a: string, b: string): boolean { + const bounds = (value: string): [number, number] => { + const range = parseIpv4Cidr(value); + if (range === undefined) { + throw new Error(`${value} is not an IPv4 CIDR range.`); + } + const size = 2 ** (32 - range.prefix); + const start = range.address - (range.address % size); + return [start, start + size]; + }; + const [startA, endA] = bounds(a); + const [startB, endB] = bounds(b); + return startA < endB && startB < endA; +} diff --git a/examples/global-router/cdk/lib/config.ts b/examples/global-router/cdk/lib/config.ts index 6f492f878df4..c0a47c49cac7 100644 --- a/examples/global-router/cdk/lib/config.ts +++ b/examples/global-router/cdk/lib/config.ts @@ -84,6 +84,35 @@ export interface PeeringRegionConfig { readonly cidrBlock: string; readonly vpcId: string; readonly privateRouteTableIds: readonly [string, ...string[]]; + /** + * Public subnet route tables. Routing them over the peering lets each + * bastion reach the other VPC, for example to join Karmada. + */ + readonly publicRouteTableIds?: readonly string[]; +} + +/** Karmada control plane, installed in the hub cluster only. */ +export interface KarmadaControlPlaneConfig { + /** + * Karmada release, which pins the chart and its images. It must be + * `KARMADA_VERSION`, the release that `crds/` comes from. + */ + readonly version: string; + /** VPC ranges allowed to reach the internal Karmada API load balancer. */ + readonly apiSourceCidrs: readonly [string, ...string[]]; +} + +/** Karmada and SIG Multicluster (MCS) support for one cluster. */ +export interface MultiClusterConfig { + /** Karmada member name. The Global Router uses it as the pool site ID. */ + readonly memberName: string; + /** + * VPC ranges of the other member clusters. Their pods may reach pods in + * this cluster, and traffic from pods here to them keeps its pod address. + */ + readonly peerCidrs: readonly [string, ...string[]]; + /** Set in the hub cluster only. */ + readonly karmadaControlPlane?: KarmadaControlPlaneConfig; } export interface DeploymentConfig { @@ -99,4 +128,6 @@ export interface DeploymentConfig { readonly dynamoPlatformConfig: DynamoPlatformConfig; readonly dgdPools: readonly [DgdPoolConfig, ...DgdPoolConfig[]]; readonly dgds: readonly [DgdConfig, ...DgdConfig[]]; + /** Karmada and MCS support; omit for a single-cluster deployment. */ + readonly multiCluster?: MultiClusterConfig; } diff --git a/examples/global-router/cdk/lib/const.ts b/examples/global-router/cdk/lib/const.ts index 2481a25490cf..b5dcc1e80c7d 100644 --- a/examples/global-router/cdk/lib/const.ts +++ b/examples/global-router/cdk/lib/const.ts @@ -30,6 +30,7 @@ import { US_EAST_2_BASTION_KEY_PAIR, US_WEST_2_BASTION_KEY_PAIR, } from "./keypairs"; +import { KARMADA_VERSION } from "./multi-cluster"; const US_EAST_2 = "us-east-2"; const US_EAST_2A = "us-east-2a"; @@ -146,6 +147,18 @@ export const US_EAST2_DEPLOYMENT = { dynamoPlatformConfig: DYNAMO_PLATFORM_1_4_0, dgdPools: [MOCKER_OHIO_1_POOL], dgds: [MOCKER_OHIO_1_DGD], + // Ohio is the Karmada hub. Member names match the Router's site IDs. + multiCluster: { + memberName: "aws-ohio", + peerCidrs: [MOCKER_VPC_US_WEST_2.cidrBlock], + karmadaControlPlane: { + version: KARMADA_VERSION, + apiSourceCidrs: [ + MOCKER_VPC_US_EAST_2.cidrBlock, + MOCKER_VPC_US_WEST_2.cidrBlock, + ], + }, + }, } satisfies DeploymentConfig; export const MOCKER_WEST_1_DGD = { @@ -181,6 +194,10 @@ export const US_WEST2_DEPLOYMENT = { dynamoPlatformConfig: DYNAMO_PLATFORM_1_4_0, dgdPools: [MOCKER_WEST_1_POOL], dgds: [MOCKER_WEST_1_DGD], + multiCluster: { + memberName: "aws-west", + peerCidrs: [MOCKER_VPC_US_EAST_2.cidrBlock], + }, } satisfies DeploymentConfig; export const DEPLOYMENTS = { @@ -189,7 +206,7 @@ export const DEPLOYMENTS = { } satisfies Record; // Example network IDs; use local settings populated from your deployed stacks. -// Include only private route tables in the peering configuration. +// Public routes enable bastion access to peers without adding internet ingress. export const POC_PEERING_OHIO = { region: US_EAST_2, cidrBlock: MOCKER_VPC_US_EAST_2.cidrBlock, @@ -200,6 +217,8 @@ export const POC_PEERING_OHIO = { "rtb-00000000000000004", // dgd-pool-slot-1Subnet1 "rtb-00000000000000005", // dgd-pool-slot-1Subnet2 ], + // publicSubnet1 and publicSubnet2: the only route tables with an internet gateway route. + publicRouteTableIds: ["rtb-0000000000000000b", "rtb-0000000000000000c"], } satisfies PeeringRegionConfig; export const POC_PEERING_WEST = { @@ -212,6 +231,8 @@ export const POC_PEERING_WEST = { "rtb-00000000000000009", // dgd-pool-slot-1Subnet1 "rtb-0000000000000000a", // dgd-pool-slot-1Subnet2 ], + // publicSubnet1 and publicSubnet2: the only route tables with an internet gateway route. + publicRouteTableIds: ["rtb-0000000000000000d", "rtb-0000000000000000e"], } satisfies PeeringRegionConfig; export type DeploymentName = keyof typeof DEPLOYMENTS; diff --git a/examples/global-router/cdk/lib/dynamo-mocker-cpu-stack.ts b/examples/global-router/cdk/lib/dynamo-mocker-cpu-stack.ts index 5fae4fa9d2d6..b3dc3d7ce0c5 100644 --- a/examples/global-router/cdk/lib/dynamo-mocker-cpu-stack.ts +++ b/examples/global-router/cdk/lib/dynamo-mocker-cpu-stack.ts @@ -33,6 +33,7 @@ import * as eks from "aws-cdk-lib/aws-eks-v2"; import { Construct } from "constructs"; import { EksAdminBastion } from "./bastion"; +import { isValidIpv4Cidr, isValidVpcCidr } from "./cidr"; import type { GlobalRouterDeployment } from "./global-router-deployment"; import { createGlobalRouterHelmValues } from "./global-router-helm-values"; import { @@ -51,6 +52,13 @@ import { PURPOSE_LABEL_KEY, PURPOSE_LABEL_VALUE, } from "./helm-values"; +import { + KARMADA_API_PORT, + KARMADA_API_SERVICE, + KARMADA_KUBECONFIG_SECRET, + KARMADA_NAMESPACE, + MultiClusterSupport, +} from "./multi-cluster"; import { MAX_DGD_POOL_SUBNET_SLOTS, MockerVpc } from "./vpc"; const DYNAMO_PLATFORM_RELEASE = "dynamo-platform"; @@ -344,6 +352,16 @@ export class DynamoMockerCpuStack extends Stack { } } + if (config.multiCluster !== undefined) { + new MultiClusterSupport(this, "MultiCluster", { + cluster, + config: config.multiCluster, + region: clusterConfig.region, + vpcCidr: clusterConfig.vpcConfig.cidrBlock, + nodeGroups: poolNodeGroups.map(({ nodeGroup }) => nodeGroup), + }); + } + Tags.of(this).add("ManagedBy", "AWS-CDK"); Tags.of(this).add("Project", "dynamo-mocker-cpu"); @@ -407,6 +425,28 @@ export class DynamoMockerCpuStack extends Stack { .join("; "), }); } + if (config.multiCluster !== undefined) { + new CfnOutput(this, "KarmadaMemberName", { + description: "Use with karmadactl register --cluster-name.", + value: config.multiCluster.memberName, + }); + if (config.multiCluster.karmadaControlPlane !== undefined) { + new CfnOutput(this, "KarmadaApiEndpointCommand", { + description: + `Run on the bastion: prints the internal Karmada API load balancer hostname (port ${KARMADA_API_PORT}).`, + value: + `kubectl get service -n ${KARMADA_NAMESPACE} ${KARMADA_API_SERVICE} ` + + "-o jsonpath='{.status.loadBalancer.ingress[0].hostname}'", + }); + new CfnOutput(this, "KarmadaKubeconfigCommand", { + description: + "Run on the bastion: writes the Karmada admin kubeconfig. Point its server at the load balancer.", + value: + `kubectl get secret -n ${KARMADA_NAMESPACE} ${KARMADA_KUBECONFIG_SECRET} ` + + "-o jsonpath='{.data.kubeconfig}' | base64 -d > karmada-apiserver.config", + }); + } + } new CfnOutput(this, "PortForwardFrontends", { description: "Semicolon-separated DGD-to-port-forward commands.", value: orderedDgds @@ -668,45 +708,6 @@ function escapeRegex(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } -function isValidIpv4Cidr(value: string): boolean { - const match = - /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\/(\d{1,2})$/.exec( - value, - ); - if (match === null) { - return false; - } - const octets = match.slice(1, 5).map(Number); - const prefix = Number(match[5]); - return !octets.some((octet) => octet > 255) && prefix <= 32; -} - -function isValidVpcCidr(value: string): boolean { - const match = - /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\/(\d{1,2})$/.exec( - value, - ); - if (match === null) { - return false; - } - const octets = match.slice(1, 5).map(Number); - const prefix = Number(match[5]); - if (octets.some((octet) => octet > 255) || prefix < 16 || prefix > 24) { - return false; - } - const address = - octets[0] * 2 ** 24 + - octets[1] * 2 ** 16 + - octets[2] * 2 ** 8 + - octets[3]; - const blockSize = 2 ** (32 - prefix); - const isPrivate = - octets[0] === 10 || - (octets[0] === 172 && octets[1] >= 16 && octets[1] <= 31) || - (octets[0] === 192 && octets[1] === 168); - return isPrivate && address % blockSize === 0; -} - function isKubernetesName(value: string, maximumLength: number): boolean { return ( value.length <= maximumLength && diff --git a/examples/global-router/cdk/lib/inter-region-peering.ts b/examples/global-router/cdk/lib/inter-region-peering.ts index 8c64e4dfe2e8..b8f2d30b41d5 100644 --- a/examples/global-router/cdk/lib/inter-region-peering.ts +++ b/examples/global-router/cdk/lib/inter-region-peering.ts @@ -21,7 +21,7 @@ import type { Construct } from "constructs"; import type { PeeringRegionConfig } from "./config"; /** - * Add inter-Region peering and private routes in dedicated stacks. Keeping the + * Add inter-Region peering and its routes in dedicated stacks. Keeping the * routes outside the VPC stacks avoids a cross-Region CloudFormation cycle. */ export function addInterRegionPeering( @@ -39,10 +39,18 @@ export function addInterRegionPeering( if (ohioVpc.vpcId === westVpc.vpcId) { throw new Error("Inter-Region VPC peering requires distinct VPC IDs."); } + for (const vpc of [ohioVpc, westVpc]) { + const publicIds = vpc.publicRouteTableIds ?? []; + if (publicIds.some((id) => vpc.privateRouteTableIds.includes(id))) { + throw new Error( + `VPC ${vpc.vpcId} lists a route table as both private and public.`, + ); + } + } const peeringRoutesStack = new Stack(scope, "OhioWestPeeringRoutes", { crossRegionReferences: true, - description: "Ohio to West VPC peering and Ohio private routes", + description: "Ohio to West VPC peering and Ohio routes", env: { account, region: ohioVpc.region }, }); const peering = new ec2.CfnVPCPeeringConnection( @@ -58,29 +66,45 @@ export function addInterRegionPeering( vpcId: ohioVpc.vpcId, }, ); - addPrivateRoutes( + addRoutes( peeringRoutesStack, ohioVpc.privateRouteTableIds, westVpc.cidrBlock, peering.attrId, - "OhioToWest", + "OhioToWestPrivate", + ); + addRoutes( + peeringRoutesStack, + ohioVpc.publicRouteTableIds ?? [], + westVpc.cidrBlock, + peering.attrId, + "OhioToWestPublic", ); + // The ID predates the public routes; renaming it would replace the stack. const westRoutesStack = new Stack(scope, "WestPrivateRoutes", { crossRegionReferences: true, - description: "West private routes to the Ohio VPC over peering", + description: "West routes to the Ohio VPC over peering", env: { account, region: westVpc.region }, }); - addPrivateRoutes( + addRoutes( westRoutesStack, westVpc.privateRouteTableIds, ohioVpc.cidrBlock, peering.attrId, - "WestToOhio", + "WestToOhioPrivate", + ); + addRoutes( + westRoutesStack, + westVpc.publicRouteTableIds ?? [], + ohioVpc.cidrBlock, + peering.attrId, + "WestToOhioPublic", ); } -function addPrivateRoutes( +/** Logical IDs are `Route`, so existing private routes keep theirs. */ +function addRoutes( stack: Stack, routeTableIds: readonly string[], destinationCidr: string, @@ -88,7 +112,7 @@ function addPrivateRoutes( idPrefix: string, ): void { for (const [index, routeTableId] of routeTableIds.entries()) { - new ec2.CfnRoute(stack, `${idPrefix}PrivateRoute${index + 1}`, { + new ec2.CfnRoute(stack, `${idPrefix}Route${index + 1}`, { destinationCidrBlock: destinationCidr, routeTableId, vpcPeeringConnectionId: peeringId, diff --git a/examples/global-router/cdk/lib/multi-cluster.ts b/examples/global-router/cdk/lib/multi-cluster.ts new file mode 100644 index 000000000000..f5db9e895b0d --- /dev/null +++ b/examples/global-router/cdk/lib/multi-cluster.ts @@ -0,0 +1,326 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as fs from "node:fs"; +import * as path from "node:path"; + +import { Duration, RemovalPolicy, aws_ec2 as ec2 } from "aws-cdk-lib"; +import * as eks from "aws-cdk-lib/aws-eks-v2"; +import { Construct, type IDependable } from "constructs"; + +import { cidrsOverlap, isValidIpv4Cidr, isValidVpcCidr } from "./cidr"; +import type { + DeploymentConfig, + KarmadaControlPlaneConfig, + MultiClusterConfig, +} from "./config"; + +export const KARMADA_NAMESPACE = "karmada-system"; +export const KARMADA_RELEASE = "karmada"; +/** The chart names the API server Service and its kubeconfig after the release. */ +export const KARMADA_API_SERVICE = `${KARMADA_RELEASE}-apiserver`; +export const KARMADA_KUBECONFIG_SECRET = `${KARMADA_RELEASE}-kubeconfig`; +export const KARMADA_API_PORT = 5443; +/** The release that `crds/` and the setup image pins below come from. */ +export const KARMADA_VERSION = "1.19.0"; + +/** `karmadactl register` rejects longer member names. */ +const MAX_MEMBER_NAME_LENGTH = 48; + +/** + * Helm must finish waiting, and roll back if it has to, well inside the + * kubectl Lambda's 15-minute limit. + */ +const KARMADA_HELM_TIMEOUT = Duration.minutes(10); + +// The release chart pins the Karmada component images, but its setup jobs +// default to `latest`. cfssl v1.6.5 was current when Karmada 1.19.0 shipped; +// the kubectl image only publishes `latest`, so it is pinned by digest. +const CFSSL_IMAGE_TAG = "v1.6.5"; +const KUBECTL_IMAGE_DIGEST = + "sha256:9b58231669c7c371ecd976626a61d72ae7e46326f30932e8934a087f5f26fbd8"; + +// The chart's default certificate names. Setting `certs.auto.hosts` replaces +// the list, so these are kept next to the load balancer's names. +const KARMADA_DEFAULT_CERT_HOSTS = [ + "kubernetes.default.svc", + "*.etcd.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}", + "*.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}", + "*.{{ .Release.Namespace }}.svc", + "localhost", + "127.0.0.1", +]; + +// From Karmada `KARMADA_VERSION`, `charts/karmada/_crds/bases/multicluster`; +// see crds/README.md. +const MCS_CRD_FILES = [ + "multicluster.x-k8s.io_serviceexports.json", + "multicluster.x-k8s.io_serviceimports.json", +]; + +const EXCLUDE_SNAT_ENV = "AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS"; + +export interface MultiClusterSupportProps { + readonly cluster: eks.Cluster; + readonly config: MultiClusterConfig; + readonly region: string; + /** This cluster's VPC range. It must not be one of the peers. */ + readonly vpcCidr: string; + /** Node groups the Karmada control plane runs on. */ + readonly nodeGroups: readonly IDependable[]; +} + +/** + * Prepares one cluster for Karmada and the Multi-Cluster Services (MCS) API: + * + * - Pods in peer VPCs may reach this cluster's pods. MCS EndpointSlices list + * pod addresses, so callers in other clusters connect to pods directly. + * - Traffic from pods here to peer VPCs keeps its pod address (no SNAT), so + * a peer can tell which pod is calling. + * - The MCS `ServiceExport` and `ServiceImport` types are installed. + * - In the hub only: the Karmada control plane, behind an internal load + * balancer that every member VPC can reach. + * + * Members join Karmada in Pull mode with `karmadactl register` (see the + * README). The join uses a short-lived token from the hub, so it is a step + * after deployment rather than a CloudFormation resource. + */ +export class MultiClusterSupport extends Construct { + constructor(scope: Construct, id: string, props: MultiClusterSupportProps) { + super(scope, id); + const { cluster, config } = props; + validateMultiClusterConfig(config, props.vpcCidr); + + for (const cidr of config.peerCidrs) { + cluster.clusterSecurityGroup.addIngressRule( + ec2.Peer.ipv4(cidr), + ec2.Port.allTcp(), + `MCS pod traffic from peer VPC ${cidr}`, + ); + // Inter-Region peering carries smaller packets than the pods' MTU, and + // senders learn the limit from this ICMP message (Path MTU Discovery). + cluster.clusterSecurityGroup.addIngressRule( + ec2.Peer.ipv4(cidr), + ec2.Port.icmpTypeAndCode(3, 4), + `Path MTU Discovery from peer VPC ${cidr}`, + ); + } + + new eks.KubernetesPatch(this, "ExcludePeerVpcsFromSnat", { + cluster, + resourceName: "daemonset/aws-node", + resourceNamespace: "kube-system", + applyPatch: awsNodeEnvPatch({ + name: EXCLUDE_SNAT_ENV, + value: config.peerCidrs.join(","), + }), + restorePatch: awsNodeEnvPatch({ + $patch: "delete", + name: EXCLUDE_SNAT_ENV, + }), + }); + + // `overwrite` applies instead of creating, so CRDs that already exist + // in a running cluster are adopted. They are kept on removal and + // rollback: they may predate this stack, and deleting a CRD deletes every + // ServiceExport and ServiceImport. + new eks.KubernetesManifest(this, "McsCrds", { + cluster, + manifest: loadMcsCrds(), + overwrite: true, + removalPolicy: RemovalPolicy.RETAIN, + }); + + const karmada = config.karmadaControlPlane; + if (karmada !== undefined) { + if (cluster.albController === undefined) { + throw new Error( + "The Karmada API load balancer needs the AWS Load Balancer Controller.", + ); + } + const chart = new eks.HelmChart(this, "KarmadaControlPlane", { + atomic: true, + chart: karmadaChartUrl(karmada.version), + cluster, + createNamespace: true, + namespace: KARMADA_NAMESPACE, + release: KARMADA_RELEASE, + // CDK never uninstalls it: the chart's uninstall hook also deletes the + // MCS CRDs and its own namespace (see the README). + removalPolicy: RemovalPolicy.RETAIN, + timeout: KARMADA_HELM_TIMEOUT, + values: createKarmadaControlPlaneValues(karmada, props.region), + // Waits for the control plane pods, not for the load balancer's + // address; the README's join steps check for that. + wait: true, + }); + chart.node.addDependency(cluster.albController); + for (const nodeGroup of props.nodeGroups) { + chart.node.addDependency(nodeGroup); + } + } + } +} + +/** + * Returns `config` without the hub's Karmada control plane unless `install` + * is set (`-c karmada=true`). The running clusters still have a Karmada that + * was installed by hand, which Helm cannot install over (see the README). + */ +export function withKarmadaControlPlane( + config: DeploymentConfig, + install: boolean, +): DeploymentConfig { + const multiCluster = config.multiCluster; + if (install) { + if (multiCluster?.karmadaControlPlane === undefined) { + throw new Error( + "-c karmada=true needs a deployment with a Karmada control plane (Ohio).", + ); + } + return config; + } + if (multiCluster?.karmadaControlPlane === undefined) { + return config; + } + const { karmadaControlPlane: _notInstalled, ...member } = multiCluster; + return { ...config, multiCluster: member }; +} + +export function validateMultiClusterConfig( + config: MultiClusterConfig, + vpcCidr: string, +): void { + if ( + config.memberName.length > MAX_MEMBER_NAME_LENGTH || + !/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/.test(config.memberName) + ) { + throw new Error( + `Karmada member name ${config.memberName} must be a lowercase DNS label of at most ${MAX_MEMBER_NAME_LENGTH} characters.`, + ); + } + if (config.peerCidrs.some((cidr) => !isValidVpcCidr(cidr))) { + throw new Error( + "Multi-cluster peer CIDRs must be private, network-aligned VPC ranges from /16 to /24.", + ); + } + if (config.peerCidrs.some((cidr) => cidrsOverlap(cidr, vpcCidr))) { + throw new Error("Multi-cluster peer CIDRs must not overlap the cluster's own VPC."); + } + const karmada = config.karmadaControlPlane; + if (karmada === undefined) { + return; + } + if (karmada.version !== KARMADA_VERSION) { + throw new Error( + `Karmada ${karmada.version} is not supported: crds/ and the pinned setup images come from ${KARMADA_VERSION}. crds/README.md says how to move them together.`, + ); + } + if (karmada.apiSourceCidrs.some((cidr) => !isValidIpv4Cidr(cidr))) { + throw new Error("Karmada API source CIDRs must be valid IPv4 CIDR ranges."); + } +} + +/** The chart that each Karmada release publishes. */ +export function karmadaChartUrl(version: string): string { + return ( + `https://github.com/karmada-io/karmada/releases/download/v${version}/` + + `karmada-chart-v${version}.tgz` + ); +} + +export function createKarmadaControlPlaneValues( + config: KarmadaControlPlaneConfig, + region: string, +): Record { + return { + installMode: "host", + cfssl: { image: { tag: CFSSL_IMAGE_TAG } }, + kubectl: { image: { digest: KUBECTL_IMAGE_DIGEST } }, + // Helm 4 waits until every resource is ready at the same time and counts + // a deleted Job as never ready, so the setup Job must outlive the wait. + staticResourceJob: { + ttlSecondsAfterFinished: KARMADA_HELM_TIMEOUT.toSeconds() + 300, + }, + certs: { + mode: "auto", + auto: { + // The internal load balancer gets a name in this pattern. + hosts: [...KARMADA_DEFAULT_CERT_HOSTS, `*.elb.${region}.amazonaws.com`], + }, + }, + apiServer: { + serviceType: "LoadBalancer", + serviceAnnotations: { + "service.beta.kubernetes.io/aws-load-balancer-type": "external", + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type": "ip", + "service.beta.kubernetes.io/aws-load-balancer-scheme": "internal", + "service.beta.kubernetes.io/load-balancer-source-ranges": + config.apiSourceCidrs.join(","), + }, + }, + // A propagated ServiceImport brings its derived Service and EndpointSlices + // only with this gate on. The chart turns it off; Karmada's own default, + // which `karmadactl init` keeps, is on. + controllerManager: { featureGates: { PropagateDeps: true } }, + }; +} + +function awsNodeEnvPatch(env: Record): Record { + return { + spec: { + template: { + spec: { + containers: [{ name: "aws-node", env: [env] }], + }, + }, + }, + }; +} + +function loadMcsCrds(): Record[] { + const directory = resolveCrdDirectory(); + return MCS_CRD_FILES.map((file) => { + const crd: unknown = JSON.parse( + fs.readFileSync(path.join(directory, file), "utf8"), + ); + if ( + crd === null || + typeof crd !== "object" || + (crd as { kind?: unknown }).kind !== "CustomResourceDefinition" + ) { + throw new Error(`crds/${file} is not a CustomResourceDefinition; see crds/README.md.`); + } + return crd as Record; + }); +} + +function resolveCrdDirectory(): string { + const candidates = [ + path.resolve(__dirname, "..", "crds"), + path.resolve(__dirname, "..", "..", "crds"), + ]; + const directory = candidates.find((candidate) => + fs.existsSync(path.join(candidate, MCS_CRD_FILES[0])), + ); + if (directory === undefined) { + throw new Error( + `Unable to locate the MCS CRDs; checked ${candidates.join(", ")}.`, + ); + } + return directory; +} diff --git a/examples/global-router/cdk/test/inter-region-peering.test.ts b/examples/global-router/cdk/test/inter-region-peering.test.ts index ec805812cb13..977ab26f8b02 100644 --- a/examples/global-router/cdk/test/inter-region-peering.test.ts +++ b/examples/global-router/cdk/test/inter-region-peering.test.ts @@ -24,7 +24,7 @@ import { Template } from "aws-cdk-lib/assertions"; import { POC_PEERING_OHIO, POC_PEERING_WEST } from "../lib/const"; import { addInterRegionPeering } from "../lib/inter-region-peering"; -test("both mode creates one Ohio peering connection and private routes both ways", () => { +test("both mode creates one Ohio peering connection and routes both ways", () => { const app = new App(); addInterRegionPeering( app, @@ -50,11 +50,74 @@ test("both mode creates one Ohio peering connection and private routes both ways { Key: "Project", Value: "dynamo-mocker-cpu" }, ], }); - assertPrivateRoutes(peeringTemplate, "10.43.0.0/16", POC_PEERING_OHIO.privateRouteTableIds); - assertPrivateRoutes(westRoutesTemplate, "10.42.0.0/16", POC_PEERING_WEST.privateRouteTableIds); + assertPeeringRoutes(peeringTemplate, "10.43.0.0/16", [ + ...POC_PEERING_OHIO.privateRouteTableIds, + ...POC_PEERING_OHIO.publicRouteTableIds, + ]); + assertPeeringRoutes(westRoutesTemplate, "10.42.0.0/16", [ + ...POC_PEERING_WEST.privateRouteTableIds, + ...POC_PEERING_WEST.publicRouteTableIds, + ]); + // Every route, public ones included, uses this peering connection; West + // reads its ID across Regions. + for (const route of Object.values(peeringTemplate.findResources("AWS::EC2::Route"))) { + assert.deepEqual(route.Properties.VpcPeeringConnectionId, { + "Fn::GetAtt": ["OhioWestVpcPeering", "Id"], + }); + } + for (const route of Object.values(westRoutesTemplate.findResources("AWS::EC2::Route"))) { + assert.match(JSON.stringify(route.Properties.VpcPeeringConnectionId), /OhioWestVpcPeeringId/); + } +}); + +test("keeps private route IDs and adds public routes only when configured", () => { + const app = new App(); + addInterRegionPeering( + app, + { ...POC_PEERING_OHIO, publicRouteTableIds: ["rtb-0hiopublic1", "rtb-0hiopublic2"] }, + { ...POC_PEERING_WEST, publicRouteTableIds: [] }, + "111111111111", + ); + const peeringStack = app.node.tryFindChild("OhioWestPeeringRoutes") as Stack; + const westRoutesStack = app.node.tryFindChild("WestPrivateRoutes") as Stack; + const ohioRoutes = Template.fromStack(peeringStack).findResources("AWS::EC2::Route"); + const westRoutes = Template.fromStack(westRoutesStack).findResources("AWS::EC2::Route"); + + // Logical IDs of deployed routes must not change, or CloudFormation replaces them. + assert.ok("OhioToWestPrivateRoute1" in ohioRoutes); + assert.ok("WestToOhioPrivateRoute1" in westRoutes); + assert.equal( + (ohioRoutes.OhioToWestPublicRoute2 as { Properties: Record }) + .Properties.RouteTableId, + "rtb-0hiopublic2", + ); + assert.equal( + Object.keys(ohioRoutes).length, + POC_PEERING_OHIO.privateRouteTableIds.length + 2, + ); + assert.equal( + Object.keys(westRoutes).length, + POC_PEERING_WEST.privateRouteTableIds.length, + ); +}); + +test("rejects a route table listed as both private and public", () => { + assert.throws( + () => + addInterRegionPeering( + new App(), + POC_PEERING_OHIO, + { + ...POC_PEERING_WEST, + publicRouteTableIds: [POC_PEERING_WEST.privateRouteTableIds[0]], + }, + "111111111111", + ), + /both private and public/, + ); }); -function assertPrivateRoutes( +function assertPeeringRoutes( template: Template, destinationCidr: string, expectedRouteTableIds: readonly string[], diff --git a/examples/global-router/cdk/test/multi-cluster.test.ts b/examples/global-router/cdk/test/multi-cluster.test.ts new file mode 100644 index 000000000000..52d85d95c0a6 --- /dev/null +++ b/examples/global-router/cdk/test/multi-cluster.test.ts @@ -0,0 +1,329 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { App } from "aws-cdk-lib"; +import { Template } from "aws-cdk-lib/assertions"; + +import { cidrsOverlap } from "../lib/cidr"; +import type { DeploymentConfig, MultiClusterConfig } from "../lib/config"; +import { + MOCKER_VPC_US_EAST_2, + MOCKER_VPC_US_WEST_2, + US_EAST2_DEPLOYMENT, + US_WEST2_DEPLOYMENT, +} from "../lib/const"; +import { DynamoMockerCpuStack } from "../lib/dynamo-mocker-cpu-stack"; +import { + KARMADA_VERSION, + karmadaChartUrl, + withKarmadaControlPlane, +} from "../lib/multi-cluster"; +import { VALID_CONFIG } from "./fixtures"; + +interface Resource { + readonly DeletionPolicy?: string; + readonly DependsOn?: readonly string[]; + readonly Properties: Record; +} + +const MEMBER: MultiClusterConfig = { + memberName: "test-west", + peerCidrs: ["10.43.0.0/16"], +}; + +const HUB: MultiClusterConfig = { + memberName: "test-hub", + peerCidrs: ["10.43.0.0/16"], + karmadaControlPlane: { + version: "1.19.0", + apiSourceCidrs: ["10.42.0.0/16", "10.43.0.0/16"], + }, +}; + +function synthesize(multiCluster?: MultiClusterConfig): Template { + const config: DeploymentConfig = { ...VALID_CONFIG, multiCluster }; + const app = new App(); + const stack = new DynamoMockerCpuStack(app, "TestStack", { + config, + env: { account: "111111111111", region: config.clusterConfig.region }, + }); + return Template.fromStack(stack); +} + +function resources(template: Template, type: string): Resource[] { + return Object.values(template.findResources(type)) as Resource[]; +} + +function karmadaChart(template: Template): Resource | undefined { + return resources(template, "Custom::AWSCDK-EKS-HelmChart").find( + (chart) => chart.Properties.Release === "karmada", + ); +} + +function mcsCrdManifests(template: Template): Resource[] { + return resources(template, "Custom::AWSCDK-EKS-KubernetesResource").filter( + (manifest) => + JSON.stringify(manifest.Properties.Manifest).includes( + "serviceexports.multicluster.x-k8s.io", + ), + ); +} + +function peerIngress(template: Template, description = "MCS pod traffic"): Resource[] { + return resources(template, "AWS::EC2::SecurityGroupIngress").filter( + (rule) => String(rule.Properties.Description).startsWith(description), + ); +} + +/** Every `tag` value anywhere in the chart values. */ +function imageTags(values: unknown): string[] { + if (Array.isArray(values)) { + return values.flatMap(imageTags); + } + if (values === null || typeof values !== "object") { + return []; + } + return Object.entries(values).flatMap(([key, value]) => + key === "tag" && typeof value === "string" ? [value] : imageTags(value), + ); +} + +test("a single-cluster deployment gets no Karmada or MCS resources", () => { + const template = synthesize(); + assert.equal(karmadaChart(template), undefined); + assert.equal(mcsCrdManifests(template).length, 0); + assert.equal(peerIngress(template).length, 0); + template.resourceCountIs("Custom::AWSCDK-EKS-KubernetesPatch", 0); + assert.equal(template.findOutputs("KarmadaMemberName").KarmadaMemberName, undefined); +}); + +test("a member admits peer pod traffic, keeps pod addresses and installs the MCS CRDs", () => { + const template = synthesize(MEMBER); + + const rules = peerIngress(template); + assert.equal(rules.length, 1); + assert.equal(rules[0].Properties.CidrIp, "10.43.0.0/16"); + assert.equal(rules[0].Properties.IpProtocol, "tcp"); + assert.deepEqual([rules[0].Properties.FromPort, rules[0].Properties.ToPort], [0, 65535]); + // Pods use their node's security groups. The node launch templates set none, + // so EKS gives the nodes the cluster security group, which this rule is on. + const [clusterId] = Object.keys(template.findResources("AWS::EKS::Cluster")); + assert.deepEqual(rules[0].Properties.GroupId, { + "Fn::GetAtt": [clusterId, "ClusterSecurityGroupId"], + }); + const nodegroups = resources(template, "AWS::EKS::Nodegroup"); + assert.ok(nodegroups.length > 0); + for (const nodegroup of nodegroups) { + const { Id } = nodegroup.Properties.LaunchTemplate as { Id: { Ref: string } }; + const data = template.toJSON().Resources[Id.Ref].Properties.LaunchTemplateData; + for (const key of ["SecurityGroupIds", "SecurityGroups", "NetworkInterfaces"]) { + assert.equal(data[key], undefined, `node launch template sets ${key}`); + } + } + // Path MTU Discovery over the peering needs ICMP "fragmentation needed". + const icmp = peerIngress(template, "Path MTU Discovery"); + assert.equal(icmp.length, 1); + assert.equal(icmp[0].Properties.CidrIp, "10.43.0.0/16"); + assert.equal(icmp[0].Properties.IpProtocol, "icmp"); + assert.deepEqual([icmp[0].Properties.FromPort, icmp[0].Properties.ToPort], [3, 4]); + assert.deepEqual(icmp[0].Properties.GroupId, rules[0].Properties.GroupId); + + const [patch] = resources(template, "Custom::AWSCDK-EKS-KubernetesPatch"); + assert.equal(patch.Properties.ResourceName, "daemonset/aws-node"); + assert.equal(patch.Properties.ResourceNamespace, "kube-system"); + // A JSON merge patch would replace the whole container list. + assert.equal(patch.Properties.PatchType, "strategic"); + const env = (patch: unknown) => + (patch as { spec: { template: { spec: { containers: Array<{ name: string; env: unknown[] }> } } } }) + .spec.template.spec.containers[0]; + const applied = env(JSON.parse(String(patch.Properties.ApplyPatchJson))); + assert.equal(applied.name, "aws-node"); + assert.deepEqual(applied.env, [ + { name: "AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS", value: "10.43.0.0/16" }, + ]); + assert.deepEqual(env(JSON.parse(String(patch.Properties.RestorePatchJson))).env, [ + { $patch: "delete", name: "AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS" }, + ]); + + const [crds] = mcsCrdManifests(template); + assert.equal(crds.Properties.Overwrite, true); + // Adopted CRDs may predate the stack; deleting one deletes every export. + assert.equal(crds.DeletionPolicy, "Retain"); + const names = (JSON.parse(String(crds.Properties.Manifest)) as Array<{ + kind: string; + metadata: { name: string }; + }>).map((crd) => `${crd.kind}/${crd.metadata.name}`); + assert.deepEqual(names, [ + "CustomResourceDefinition/serviceexports.multicluster.x-k8s.io", + "CustomResourceDefinition/serviceimports.multicluster.x-k8s.io", + ]); + + assert.equal(karmadaChart(template), undefined); + template.hasOutput("KarmadaMemberName", { Value: "test-west" }); + assert.equal( + template.findOutputs("KarmadaApiEndpointCommand").KarmadaApiEndpointCommand, + undefined, + ); +}); + +test("the hub runs a pinned Karmada control plane behind an internal load balancer", () => { + const template = synthesize(HUB); + const chart = karmadaChart(template); + assert.ok(chart); + assert.equal(chart.Properties.Chart, karmadaChartUrl("1.19.0")); + assert.equal( + chart.Properties.Chart, + "https://github.com/karmada-io/karmada/releases/download/v1.19.0/karmada-chart-v1.19.0.tgz", + ); + assert.equal(chart.Properties.Namespace, "karmada-system"); + assert.equal(chart.Properties.CreateNamespace, true); + assert.equal(chart.Properties.Wait, true); + // Helm must time out, and roll back, inside the kubectl Lambda's 15 minutes. + assert.equal(chart.Properties.Timeout, "600s"); + // The chart's uninstall hook deletes the MCS CRDs and its namespace. + assert.equal(chart.DeletionPolicy, "Retain"); + + const values = JSON.parse(String(chart.Properties.Values)) as { + installMode: string; + cfssl: { image: { tag: string } }; + kubectl: { image: { tag?: string; digest: string } }; + staticResourceJob: { ttlSecondsAfterFinished: number }; + certs: { auto: { hosts: string[] } }; + apiServer: { serviceType: string; serviceAnnotations: Record }; + controllerManager: { featureGates: Record }; + }; + assert.equal(values.installMode, "host"); + // Helm 4 counts a deleted Job as never ready, so the setup Job must + // outlive Helm's wait. + assert.ok(values.staticResourceJob.ttlSecondsAfterFinished > 600); + // A propagated ServiceImport brings its derived Service only with this on. + assert.equal(values.controllerManager.featureGates.PropagateDeps, true); + assert.equal(values.cfssl.image.tag, "v1.6.5"); + assert.match(values.kubectl.image.digest, /^sha256:[0-9a-f]{64}$/); + assert.equal(values.kubectl.image.tag, undefined); + assert.equal(values.apiServer.serviceType, "LoadBalancer"); + assert.deepEqual(values.apiServer.serviceAnnotations, { + "service.beta.kubernetes.io/aws-load-balancer-type": "external", + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type": "ip", + "service.beta.kubernetes.io/aws-load-balancer-scheme": "internal", + "service.beta.kubernetes.io/load-balancer-source-ranges": "10.42.0.0/16,10.43.0.0/16", + }); + // The load balancer's name must be valid for the API certificate, and + // the chart's own names must stay. + assert.ok(values.certs.auto.hosts.includes("*.elb.us-west-2.amazonaws.com")); + assert.ok(values.certs.auto.hosts.includes("*.{{ .Release.Namespace }}.svc")); + assert.ok(values.certs.auto.hosts.includes("localhost")); + for (const tag of imageTags(values)) { + assert.ok(tag !== "latest", "every image the chart would pull as latest is pinned"); + } + + // Waits for the node groups and the load balancer controller. + const charts = Object.entries(template.findResources("Custom::AWSCDK-EKS-HelmChart")); + const controller = charts.find( + ([, resource]) => resource.Properties.Release === "aws-load-balancer-controller", + ); + assert.ok(controller); + assert.ok(chart.DependsOn?.includes(controller[0])); + const nodegroupIds = Object.keys(template.findResources("AWS::EKS::Nodegroup")); + assert.ok(nodegroupIds.length > 0); + for (const nodegroupId of nodegroupIds) { + assert.ok(chart.DependsOn?.includes(nodegroupId), `waits for ${nodegroupId}`); + } + template.hasOutput("KarmadaMemberName", { Value: "test-hub" }); + template.hasOutput("KarmadaApiEndpointCommand", { + Value: + "kubectl get service -n karmada-system karmada-apiserver " + + "-o jsonpath='{.status.loadBalancer.ingress[0].hostname}'", + }); + template.hasOutput("KarmadaKubeconfigCommand", { + Value: + "kubectl get secret -n karmada-system karmada-kubeconfig " + + "-o jsonpath='{.data.kubeconfig}' | base64 -d > karmada-apiserver.config", + }); +}); + +test("rejects invalid multi-cluster settings", () => { + const cases: Array<[string, MultiClusterConfig, RegExp]> = [ + ["upper-case member name", { ...MEMBER, memberName: "West" }, /lowercase DNS label/], + ["long member name", { ...MEMBER, memberName: "w".repeat(49) }, /at most 48/], + ["bad peer range", { ...MEMBER, peerCidrs: ["10.43.0.0"] }, /VPC ranges/], + ["peer wider than a VPC", { ...MEMBER, peerCidrs: ["10.0.0.0/8"] }, /VPC ranges/], + ["public peer", { ...MEMBER, peerCidrs: ["8.8.0.0/16"] }, /VPC ranges/], + ["unaligned peer", { ...MEMBER, peerCidrs: ["10.43.1.0/16"] }, /VPC ranges/], + // The VPC CNI skips a zero-padded range, so its traffic would still be NATed. + ["zero-padded peer", { ...MEMBER, peerCidrs: ["010.043.000.000/16"] }, /VPC ranges/], + // VALID_CONFIG's own VPC is 10.42.0.0/16. + ["own VPC as a peer", { ...MEMBER, peerCidrs: ["10.42.0.0/16"] }, /own VPC/], + ["peer inside the own VPC", { ...MEMBER, peerCidrs: ["10.42.128.0/17"] }, /own VPC/], + [ + "inexact Karmada version", + { ...HUB, karmadaControlPlane: { ...HUB.karmadaControlPlane!, version: "1.19" } }, + /not supported/, + ], + [ + "Karmada version without matching CRDs", + { ...HUB, karmadaControlPlane: { ...HUB.karmadaControlPlane!, version: "1.18.0" } }, + /not supported/, + ], + [ + "bad API source range", + { ...HUB, karmadaControlPlane: { ...HUB.karmadaControlPlane!, apiSourceCidrs: ["any"] } }, + /API source CIDRs/, + ], + ]; + for (const [name, multiCluster, message] of cases) { + assert.throws(() => synthesize(multiCluster), message, name); + } +}); + +test("Ohio is the Karmada hub and West a member, each peering with the other", () => { + const ohio = US_EAST2_DEPLOYMENT.multiCluster; + const west = US_WEST2_DEPLOYMENT.multiCluster; + assert.equal(ohio.memberName, "aws-ohio"); + assert.deepEqual(ohio.peerCidrs, [MOCKER_VPC_US_WEST_2.cidrBlock]); + assert.deepEqual(ohio.karmadaControlPlane.apiSourceCidrs, [ + MOCKER_VPC_US_EAST_2.cidrBlock, + MOCKER_VPC_US_WEST_2.cidrBlock, + ]); + assert.equal(ohio.karmadaControlPlane.version, KARMADA_VERSION); + assert.equal(west.memberName, "aws-west"); + assert.deepEqual(west.peerCidrs, [MOCKER_VPC_US_EAST_2.cidrBlock]); + assert.equal("karmadaControlPlane" in west, false); +}); + +test("the Karmada control plane is installed only with -c karmada=true", () => { + const hub: DeploymentConfig = { ...VALID_CONFIG, multiCluster: HUB }; + const withoutFlag = withKarmadaControlPlane(hub, false); + assert.equal(withoutFlag.multiCluster?.karmadaControlPlane, undefined); + assert.deepEqual(withoutFlag.multiCluster?.peerCidrs, HUB.peerCidrs); + assert.equal(withKarmadaControlPlane(hub, true), hub); + + const member: DeploymentConfig = { ...VALID_CONFIG, multiCluster: MEMBER }; + assert.equal(withKarmadaControlPlane(member, false), member); + assert.throws(() => withKarmadaControlPlane(member, true), /karmada=true/); + assert.throws(() => withKarmadaControlPlane(VALID_CONFIG, true), /karmada=true/); +}); + +test("peer ranges overlap only when they share an address", () => { + assert.equal(cidrsOverlap("10.42.0.0/16", "10.42.128.0/17"), true); + assert.equal(cidrsOverlap("10.0.0.0/8", "10.42.0.0/16"), true); + assert.equal(cidrsOverlap("10.42.0.0/16", "10.43.0.0/16"), false); + assert.throws(() => cidrsOverlap("10.42.0.0", "10.43.0.0/16"), /not an IPv4 CIDR/); +}); From 4afd75b7ea5c51e8c063c4a1a93b48a2b2b80115 Mon Sep 17 00:00:00 2001 From: Sachal Date: Sun, 25 Oct 2026 00:51:43 -0700 Subject: [PATCH 3/3] feat(global-router): validate MCS across four existing regional clusters (#17271) Signed-off-by: Anna Tchernych Signed-off-by: Sachal Malick Co-authored-by: Anna Tchernych Co-authored-by: Claude Opus 5.5 (1M context) --- examples/global-router/cdk/MCS_FLEET.md | 161 ++++++++++++ examples/global-router/cdk/README.md | 24 +- examples/global-router/cdk/bin/mcs-fleet.ts | 51 ++++ .../cdk/lib/deployment-settings.ts | 51 +++- examples/global-router/cdk/lib/mcs-fleet.ts | 244 ++++++++++++++++++ .../global-router/cdk/lib/multi-cluster.ts | 2 +- .../global-router/cdk/test/mcs-fleet.test.ts | 152 +++++++++++ 7 files changed, 668 insertions(+), 17 deletions(-) create mode 100644 examples/global-router/cdk/MCS_FLEET.md create mode 100644 examples/global-router/cdk/bin/mcs-fleet.ts create mode 100644 examples/global-router/cdk/lib/mcs-fleet.ts create mode 100644 examples/global-router/cdk/test/mcs-fleet.test.ts diff --git a/examples/global-router/cdk/MCS_FLEET.md b/examples/global-router/cdk/MCS_FLEET.md new file mode 100644 index 000000000000..d0e86e3a5f0a --- /dev/null +++ b/examples/global-router/cdk/MCS_FLEET.md @@ -0,0 +1,161 @@ + + +# MCS on existing regional clusters + +The `bin/mcs-fleet.ts` entry point extends the Karmada/MCS support from +[PR #16963](https://github.com/ai-dynamo/dynamo/pull/16963) to two through four +existing regional clusters. It imports each cluster's private kubectl provider, +security group and route tables. It never owns worker nodes, GPU reservations, +cluster foundations or the running Karmada control plane. + +Use the existing Ohio hub when extending a running fleet. Do not run `karmadactl +init`, `deinit` or the chart migration steps over an existing installation. +The original CPU entry point still supports a fresh CDK-managed hub, but this +entry point does not install or adopt its chart. A hostPath-backed existing etcd +remains dependent on its current node; preserving it does not provide database HA. + +## Operator configuration + +Create `config/mcs-fleet.local.json`, which Git and Docker ignore. The top-level +object has `members` and `links`. Keep actual deployment identifiers out of +checked-in examples. Run with the same account and Node.js prerequisites as +[Reserved GPU compute](GPU_COMPUTE.md); its S3 asset stacks must exist in each region. +This entry point uses the current operator session without creating deployment roles. + +Each member supplies: + +| Field | Meaning | +|---|---| +| `id` | Stable alphanumeric configuration key, such as `ohio` | +| `siteId` | Karmada member name, matching the Router's site identity | +| `region`, `clusterName` | Existing EKS cluster location and name | +| `vpcId`, `cidr` | Existing VPC and its nonoverlapping private CIDR | +| `routeTableIds` | Ordered list of private/public subnet route tables needing peer access | +| `clusterSecurityGroupId` | Existing EKS cluster security group used by the nodes | +| `kubectlServiceToken` | Existing CDK kubectl framework Lambda ARN for that cluster | +| `hubApiService` | Hub only: `{ "namespace": "karmada-system", "name": "karmada-apiserver" }` | + +Preserve member keys, route-table ordering and link ownership across redeployments. +Inspect existing rules before deployment: this app must not duplicate security group +rules or routes already owned elsewhere. VPC peering is not transitive: a four-member +fleet requires six direct links. Specify every unordered member pair exactly once. + +For an existing connection managed by another stack: + +```json +{ "left": "ohio", "right": "west", "peeringId": "pcx-00000000000000000", "external": true } +``` + +An external link leaves both its connection and all its routes with their existing +owner. Verify its current private pod routes; this app does not repair external links. +For a new connection, omit `external` and initially omit `peeringId`: + +```json +{ "left": "ohio", "right": "tokyo" } +``` + +The left member owns the new peering. After creating it, copy its output ID into +that link's `peeringId`. Keep it non-external: this stack still owns the connection. + +## Deploy the three independent phases + +Install locked dependencies, build and run the CDK tests. Select the member owning +each new link and deploy `mcsMode=peerings`. Copy the outputs into local settings, +then deploy `mcsMode=routes` for each member. Literal peering IDs avoid cross-region +CloudFormation export dependencies. + +```bash +export CDK_DEFAULT_ACCOUNT=$(aws sts get-caller-identity --query Account --output text) +pnpm exec cdk deploy --app 'node --require ts-node/register bin/mcs-fleet.ts' \ + -c mcsConfigPath=config/mcs-fleet.local.json -c mcsMember=ohio -c mcsMode=peerings +``` + +Run the same command with `mcsMode=routes` after filling the IDs. Review each +changeset: routes must add private peer CIDRs through the intended peering, with +no default-route, gateway or existing destination replacement. Existing Ohio/West +network stacks are independent and are not deployed by this app. + +Deploy `mcsMode=member` for each member. It installs retained MCS CRDs, excludes +peer CIDRs from CNI source NAT, and adds peer-only TCP and ICMP Path MTU Discovery +rules. CNI DaemonSet pods roll; worker instances remain in place. The hub's existing +API Service source ranges expand to the configured VPCs while remaining internal. +No live Relay propagation policies or registrations are changed. + +All new stacks enable deletion protection. Retained network and Kubernetes resources +require deliberate cleanup; a stack destroy does not undo this fleet configuration. + +## Register new members + +Use `karmadactl` 1.19.0 and the existing hub's working internal API endpoint. A saved +kubeconfig may still point to an expired localhost port-forward: check `/readyz` +using a private temporary copy pointed at the internal NLB first. Label hub-only +coordination and independently managed test namespaces with +`namespace.karmada.io/skip-auto-propagation=true` before creating them on the +Karmada API, so its namespace controller does not try to adopt existing member namespaces. +Keep credentials +on the administration host and do not print bootstrap tokens or commit kubeconfigs. + +Before registering with `--namespace=karmada-agent`, create that namespace in the +Karmada API as well as in the member cluster. The agent elects its leader against +the Karmada API; a missing lease namespace leaves an otherwise running agent +`Ready=Unknown` with `ClusterStatusNeverUpdated`. For example: + +```bash +kubectl --kubeconfig karmada-apiserver.config apply -f - <<'YAML' +apiVersion: v1 +kind: Namespace +metadata: + name: karmada-agent + labels: + namespace.karmada.io/skip-auto-propagation: "true" +YAML +``` + +Create a short-lived registration token on the hub, then run the printed register +command with the new member's kubeconfig, `--cluster-name=`, region/provider +metadata and `--namespace=karmada-agent`. Use CA-hash verification and certificate +rotation. For a custom agent namespace, certificate rotation also needs the matching Secret +namespace flag. Karmadactl 1.19 does not set it automatically. After registration, +append it once to the generated agent command: + +```bash +kubectl -n karmada-agent patch deployment karmada-agent --type=json -p \ + '[{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--karmada-kubeconfig-namespace=karmada-agent"}]' +``` + +Delete the bootstrap token after registration. If temporary operator +access is needed, bound registration with a timeout and revoke that access in a +finally path immediately afterward, including on failure. Worker roles do not need +cluster-admin access. + +## Validate discovery and the data plane + +Use an isolated namespace and policies selecting the four explicit members. Do not +edit production Relay policies to run a test. Create each test Service directly on its source member. Propagate its ServiceExport +from Karmada to the explicit source members, without propagating the Service itself. +Karmada 1.19 initializes its export informer after an applied ServiceExport Work; +installing CRDs and creating only local exports is insufficient on a fresh agent. +If adopting an already-created test export, scope `conflictResolution: Overwrite` +to that ServiceExport policy only. +Create the matching ServiceImport and PropagationPolicy on the hub with dependency +propagation enabled. Check: + +- All four Karmada members are Pull and Ready. +- The hub collects each exported endpoint and every importing member receives all + four pod addresses in its derived Service's EndpointSlices. +- Each member can reach the other three pod addresses: twelve directed remote paths. +- Each member can use its local `derived-..svc.cluster.local` + name to reach the imported backends. +- GPU node IDs and existing hub/member workloads remain unchanged and healthy. + +This setup does not install a `clusterset.local` DNS integration. The EKS service +CIDRs may overlap because derived Services use cluster-local IP allocation; do not +route a remote cluster's Service IP or Service CIDR. Direct cross-region paths use +the nonoverlapping VPC pod addresses. + +Remove test clients, imports/policies and server exports after recording results. +Leave the member agents, CRDs, private network and existing hub installed. Actual +Global Router discovery integration and model-serving benchmarks are separate tests. diff --git a/examples/global-router/cdk/README.md b/examples/global-router/cdk/README.md index 1b8ffb8570e9..a0597796410e 100644 --- a/examples/global-router/cdk/README.md +++ b/examples/global-router/cdk/README.md @@ -359,6 +359,20 @@ karmadactl register --cluster-name aws-ohio --namesp To remove Ohio later, pass the same `--namespace karmada-agent` to `karmadactl unregister`. With the default namespace, it would delete the Secret that the control plane uses. +For an agent registered with `--namespace karmada-agent`, its leader-election lease +uses that namespace on the Karmada API too. Create it there before checking readiness: + +```bash +kubectl --kubeconfig karmada-apiserver.config apply -f - <<'YAML' +apiVersion: v1 +kind: Namespace +metadata: + name: karmada-agent + labels: + namespace.karmada.io/skip-auto-propagation: "true" +YAML +``` + Then, on the Ohio bastion, both members should be `Pull` and `Ready`: ```bash @@ -367,8 +381,10 @@ kubectl --kubeconfig karmada-apiserver.config get clusters ### Export and import Services -Create a Service and its `ServiceExport` directly in the cluster that runs the pods, not -through Karmada propagation. Kubernetes copies a Service's labels onto its EndpointSlices, so a +Create the Service directly in the cluster that runs the pods. Propagate its +`ServiceExport` from Karmada to that source cluster, without propagating the Service. +Karmada 1.19 starts the export informer after an applied Work containing a ServiceExport; +a fresh agent will not discover local-only exports until such a Work exists. Kubernetes copies a Service's labels onto its EndpointSlices, so a Service that Karmada propagates gives its slices Karmada's labels. In a test with Karmada 1.19.0, deleting the `ServiceExport` of such a Service also made `karmada-agent` delete the Service's EndpointSlice, and exporting it again found no slice until the Service changed. @@ -896,3 +912,7 @@ separately only after confirming that no other stack uses its assets and roles. ## Reserved GPU capacity See [Reserved GPU compute](GPU_COMPUTE.md) for independent compute stacks that target externally owned Capacity Reservations. + +## Existing four-region MCS fleet + +See [MCS on existing regional clusters](MCS_FLEET.md) for additive networking and member setup that preserves the running hub and reserved GPU nodes. diff --git a/examples/global-router/cdk/bin/mcs-fleet.ts b/examples/global-router/cdk/bin/mcs-fleet.ts new file mode 100644 index 000000000000..4f3da5cd5d5f --- /dev/null +++ b/examples/global-router/cdk/bin/mcs-fleet.ts @@ -0,0 +1,51 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { readFileSync } from "node:fs"; +import { App, CliCredentialsStackSynthesizer } from "aws-cdk-lib"; +import { + McsFleet, + McsMemberStack, + McsPeeringsStack, + McsRoutesStack, + validateMcsFleet, +} from "../lib/mcs-fleet"; + +const app = new App(); +const file: unknown = app.node.tryGetContext("mcsConfigPath"); +if (typeof file !== "string" || !file.endsWith(".local.json")) + throw new Error("Select ignored mcsConfigPath=config/mcs-fleet.local.json"); +const fleet: McsFleet = JSON.parse(readFileSync(file, "utf8")); +validateMcsFleet(fleet); +const member = fleet.members.find( + (m) => m.id === app.node.tryGetContext("mcsMember"), +); +if (!member) throw new Error("Choose a configured mcsMember"); +const mode = app.node.tryGetContext("mcsMode"); +const props = { + fleet, + member, + env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: member.region }, + stackName: `global-router-mcs-${member.id}-${mode}`, + synthesizer: new CliCredentialsStackSynthesizer({ + fileAssetsBucketName: `global-router-cdk-assets-${process.env.CDK_DEFAULT_ACCOUNT}-${member.region}`, + }), +}; +if (mode === "peerings") new McsPeeringsStack(app, "McsPeerings", props); +else if (mode === "routes") new McsRoutesStack(app, "McsRoutes", props); +else if (mode === "member") new McsMemberStack(app, "McsMember", props); +else throw new Error("Choose mcsMode=peerings, routes or member"); diff --git a/examples/global-router/cdk/lib/deployment-settings.ts b/examples/global-router/cdk/lib/deployment-settings.ts index b4f828efdf5c..2db73fa2163b 100644 --- a/examples/global-router/cdk/lib/deployment-settings.ts +++ b/examples/global-router/cdk/lib/deployment-settings.ts @@ -45,26 +45,45 @@ export function loadDeploymentSettings( } let settings: unknown; try { - settings = JSON.parse(fs.readFileSync(path.resolve(cwd, configPathValue), "utf8")); + settings = JSON.parse( + fs.readFileSync(path.resolve(cwd, configPathValue), "utf8"), + ); } catch (error) { - throw new Error("Could not read deployment settings JSON.", { cause: error }); + throw new Error("Could not read deployment settings JSON.", { + cause: error, + }); } - if (!isObject(settings) || !isObject(settings.deployments) - || Object.keys(settings.deployments).length === 0 - || !Object.values(settings.deployments).every(isObject) - || !isObject(settings.peering) - || !isObject(settings.peering.ohio) || !isObject(settings.peering.west)) { - throw new Error("Deployment settings must contain deployments and peering.ohio/peering.west objects."); + if ( + !isObject(settings) || + !isObject(settings.deployments) || + Object.keys(settings.deployments).length === 0 || + !Object.values(settings.deployments).every(isObject) || + !isObject(settings.peering) || + !isObject(settings.peering.ohio) || + !isObject(settings.peering.west) + ) { + throw new Error( + "Deployment settings must contain deployments and peering.ohio/peering.west objects.", + ); } // The stack validates deployment fields; peering fields are checked here // because those stacks do not pass through DynamoMockerCpuStack. for (const region of [settings.peering.ohio, settings.peering.west]) { - if (typeof region.region !== "string" || typeof region.cidrBlock !== "string" - || typeof region.vpcId !== "string" || !/^vpc-[0-9a-f]{8,17}$/.test(region.vpcId) - || !Array.isArray(region.privateRouteTableIds) || region.privateRouteTableIds.length === 0 - || !region.privateRouteTableIds.every((id: unknown) => - typeof id === "string" && /^rtb-[0-9a-f]{8,17}$/.test(id))) { - throw new Error("Each peering region must specify region, cidrBlock, vpcId and privateRouteTableIds."); + if ( + typeof region.region !== "string" || + typeof region.cidrBlock !== "string" || + typeof region.vpcId !== "string" || + !/^vpc-[0-9a-f]{8,17}$/.test(region.vpcId) || + !Array.isArray(region.privateRouteTableIds) || + region.privateRouteTableIds.length === 0 || + !region.privateRouteTableIds.every(isRouteTableId) || + (region.publicRouteTableIds !== undefined && + (!Array.isArray(region.publicRouteTableIds) || + !region.publicRouteTableIds.every(isRouteTableId))) + ) { + throw new Error( + "Each peering region must specify region, cidrBlock, vpcId and privateRouteTableIds.", + ); } } return settings as unknown as DeploymentSettings; @@ -73,3 +92,7 @@ export function loadDeploymentSettings( function isObject(value: unknown): value is Record { return value !== null && typeof value === "object" && !Array.isArray(value); } + +function isRouteTableId(id: unknown): boolean { + return typeof id === "string" && /^rtb-[0-9a-f]{8,17}$/.test(id); +} diff --git a/examples/global-router/cdk/lib/mcs-fleet.ts b/examples/global-router/cdk/lib/mcs-fleet.ts new file mode 100644 index 000000000000..f3fa596e8492 --- /dev/null +++ b/examples/global-router/cdk/lib/mcs-fleet.ts @@ -0,0 +1,244 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + CfnOutput, + CfnResource, + RemovalPolicy, + Stack, + StackProps, + aws_ec2 as ec2, +} from "aws-cdk-lib"; +import * as eks from "aws-cdk-lib/aws-eks-v2"; +import { Construct } from "constructs"; +import { cidrsOverlap, isValidVpcCidr } from "./cidr"; +import { MultiClusterSupport } from "./multi-cluster"; + +export interface McsMember { + readonly id: string; + readonly siteId: string; + readonly region: string; + readonly clusterName: string; + readonly vpcId: string; + readonly cidr: string; + readonly routeTableIds: readonly string[]; + readonly clusterSecurityGroupId: string; + readonly kubectlServiceToken: string; + /** Existing hub only. Changes API source ranges, never installs or replaces its control plane. */ + readonly hubApiService?: { + readonly name: string; + readonly namespace: string; + }; +} +export interface McsLink { + readonly left: string; + readonly right: string; + readonly peeringId?: string; + /** Another stack owns both this connection and its routes. */ + readonly external?: boolean; +} +export interface McsFleet { + readonly members: readonly McsMember[]; + readonly links: readonly McsLink[]; +} + +export function validateMcsFleet(f: McsFleet): void { + if ( + !Array.isArray(f?.members) || + f.members.length < 2 || + f.members.length > 4 || + !Array.isArray(f.links) + ) + throw new Error("Configure two to four MCS members and their links"); + const ids = new Set(), + sites = new Set(), + regions = new Set(); + for (const m of f.members) { + if ( + !/^[a-z][a-z0-9]{0,20}$/.test(m.id) || + ids.has(m.id) || + sites.has(m.siteId) || + regions.has(m.region) + ) + throw new Error("Member IDs, site IDs and regions must be unique"); + ids.add(m.id); + sites.add(m.siteId); + regions.add(m.region); + if ( + !/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/.test(m.siteId) || + m.siteId.length > 48 || + !/^[a-z]{2}-[a-z]+-\d$/.test(m.region) || + !isValidVpcCidr(m.cidr) + ) + throw new Error("Invalid member identity, region or private VPC CIDR"); + if ( + !/^vpc-[a-f0-9]{8,17}$/.test(m.vpcId) || + !/^sg-[a-f0-9]{8,17}$/.test(m.clusterSecurityGroupId) || + !m.clusterName || + !m.kubectlServiceToken.startsWith(`arn:aws:lambda:${m.region}:`) + ) + throw new Error( + "Specify existing cluster, VPC, security group and regional kubectl provider", + ); + if ( + !m.routeTableIds.length || + new Set(m.routeTableIds).size !== m.routeTableIds.length || + m.routeTableIds.some((id: string) => !/^rtb-[a-f0-9]{8,17}$/.test(id)) + ) + throw new Error("Specify unique regional route table IDs"); + } + if (f.members.filter((m) => m.hubApiService).length > 1) + throw new Error("Configure at most one existing hub"); + const expected = new Set(); + for (let i = 0; i < f.members.length; i++) + for (let j = i + 1; j < f.members.length; j++) { + const a = f.members[i], + b = f.members[j]; + if (cidrsOverlap(a.cidr, b.cidr) || a.vpcId === b.vpcId) + throw new Error("Member VPCs must not overlap"); + expected.add([a.id, b.id].sort().join(":")); + } + const actual = new Set(); + for (const l of f.links) { + const key = [l.left, l.right].sort().join(":"); + if (!expected.has(key) || actual.has(key)) + throw new Error("Invalid or duplicate peering link"); + actual.add(key); + if (l.peeringId && !/^pcx-[a-f0-9]{8,17}$/.test(l.peeringId)) + throw new Error("Invalid peering ID"); + if (l.external && !l.peeringId) + throw new Error("External links require an existing peering ID"); + } + if (actual.size !== expected.size) + throw new Error( + "MCS direct pod connectivity requires every member pair; VPC peering is not transitive", + ); +} + +/** New peerings only; existing connections stay with their original stack. */ +export class McsPeeringsStack extends Stack { + constructor( + scope: Construct, + id: string, + props: StackProps & { fleet: McsFleet; member: McsMember }, + ) { + super(scope, id, { ...props, terminationProtection: true }); + for (const l of props.fleet.links.filter( + (l) => l.left === props.member.id && !l.external, + )) { + const right = props.fleet.members.find((m) => m.id === l.right)!; + const connection = new ec2.CfnVPCPeeringConnection( + this, + `To${right.id}`, + { + vpcId: props.member.vpcId, + peerVpcId: right.vpcId, + peerRegion: right.region, + tags: [{ key: "Project", value: "global-router-mcs" }], + }, + ); + connection.applyRemovalPolicy(RemovalPolicy.RETAIN); + new CfnOutput(this, `Peering${props.member.id}To${right.id}`, { + value: connection.ref, + }); + } + } +} + +/** Literal peering IDs from the first phase avoid cross-region export/provider dependencies. */ +export class McsRoutesStack extends Stack { + constructor( + scope: Construct, + id: string, + props: StackProps & { fleet: McsFleet; member: McsMember }, + ) { + super(scope, id, { ...props, terminationProtection: true }); + for (const l of props.fleet.links.filter( + (l) => !l.external && [l.left, l.right].includes(props.member.id), + )) { + if (!l.peeringId) + throw new Error( + "Deploy peerings and populate peering IDs before routes", + ); + const other = props.fleet.members.find( + (m) => m.id === (l.left === props.member.id ? l.right : l.left), + )!; + for (const [index, rt] of props.member.routeTableIds.entries()) { + const route = new ec2.CfnRoute(this, `To${other.id}Route${index + 1}`, { + routeTableId: rt, + destinationCidrBlock: other.cidr, + vpcPeeringConnectionId: l.peeringId, + }); + route.applyRemovalPolicy(RemovalPolicy.RETAIN); + } + } + } +} + +/** Add MCS to existing clusters without owning their compute or their Karmada control plane. */ +export class McsMemberStack extends Stack { + constructor( + scope: Construct, + id: string, + props: StackProps & { fleet: McsFleet; member: McsMember }, + ) { + super(scope, id, { ...props, terminationProtection: true }); + const m = props.member; + const provider = eks.KubectlProvider.fromKubectlProviderAttributes( + this, + "Provider", + { serviceToken: m.kubectlServiceToken }, + ); + const cluster = eks.Cluster.fromClusterAttributes(this, "Cluster", { + clusterName: m.clusterName, + clusterSecurityGroupId: m.clusterSecurityGroupId, + kubectlProvider: provider, + }); + const peers = props.fleet.members + .filter((p) => p.id !== m.id) + .map((p) => p.cidr) as [string, ...string[]]; + new MultiClusterSupport(this, "Mcs", { + cluster, + config: { memberName: m.siteId, peerCidrs: peers }, + region: m.region, + vpcCidr: m.cidr, + nodeGroups: [], + }); + if (m.hubApiService) { + const cidrs = props.fleet.members.map((p) => p.cidr); + new eks.KubernetesPatch(this, "HubPrivateApiSources", { + cluster, + resourceName: `service/${m.hubApiService.name}`, + resourceNamespace: m.hubApiService.namespace, + applyPatch: { + spec: { loadBalancerSourceRanges: cidrs }, + metadata: { + annotations: { + "service.beta.kubernetes.io/load-balancer-source-ranges": + cidrs.join(","), + }, + }, + }, + restorePatch: {}, + }); + } + // These resources may predate this stack. Never remove their CRDs, CNI settings or peer access on rollback. + for (const c of this.node.findAll()) + if (c instanceof CfnResource) c.applyRemovalPolicy(RemovalPolicy.RETAIN); + new CfnOutput(this, "MemberName", { value: m.siteId }); + } +} diff --git a/examples/global-router/cdk/lib/multi-cluster.ts b/examples/global-router/cdk/lib/multi-cluster.ts index f5db9e895b0d..c9758797b89a 100644 --- a/examples/global-router/cdk/lib/multi-cluster.ts +++ b/examples/global-router/cdk/lib/multi-cluster.ts @@ -75,7 +75,7 @@ const MCS_CRD_FILES = [ const EXCLUDE_SNAT_ENV = "AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS"; export interface MultiClusterSupportProps { - readonly cluster: eks.Cluster; + readonly cluster: eks.ICluster & { readonly albController?: IDependable }; readonly config: MultiClusterConfig; readonly region: string; /** This cluster's VPC range. It must not be one of the peers. */ diff --git a/examples/global-router/cdk/test/mcs-fleet.test.ts b/examples/global-router/cdk/test/mcs-fleet.test.ts new file mode 100644 index 000000000000..07e481a382cf --- /dev/null +++ b/examples/global-router/cdk/test/mcs-fleet.test.ts @@ -0,0 +1,152 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { App } from "aws-cdk-lib"; +import { Template } from "aws-cdk-lib/assertions"; +import { + McsFleet, + McsMemberStack, + McsPeeringsStack, + McsRoutesStack, + validateMcsFleet, +} from "../lib/mcs-fleet"; + +const names = ["ohio", "west", "tokyo", "saopaulo"]; +const regions = ["us-east-2", "us-west-2", "ap-northeast-1", "sa-east-1"]; +const fleet: McsFleet = { + members: names.map((id, i) => ({ + id, + siteId: `aws-${id}`, + region: regions[i], + clusterName: `example-${id}`, + vpcId: `vpc-0000000000000000${i}`, + cidr: `10.${42 + i}.0.0/16`, + routeTableIds: [`rtb-0000000000000000${i}`, `rtb-0000000000000001${i}`], + clusterSecurityGroupId: `sg-0000000000000000${i}`, + kubectlServiceToken: `arn:aws:lambda:${regions[i]}:111111111111:function:example-kubectl`, + })), + links: names.flatMap((left, i) => + names + .slice(i + 1) + .map((right, j) => ({ + left, + right, + peeringId: `pcx-000000000000000${i}${j}`, + external: left === "ohio" && right === "west", + })), + ), +}; +const env = { account: "111111111111", region: "us-east-2" }; + +test("four members require six direct links and nonoverlapping private pod networks", () => { + validateMcsFleet(fleet); + assert.throws( + () => validateMcsFleet({ ...fleet, links: fleet.links.slice(1) }), + /every member pair/, + ); + assert.throws( + () => + validateMcsFleet({ ...fleet, links: [...fleet.links, fleet.links[0]] }), + /duplicate/, + ); + assert.throws( + () => + validateMcsFleet({ + ...fleet, + members: fleet.members.map((m, i) => + i === 1 ? { ...m, cidr: fleet.members[0].cidr } : m, + ), + }), + /overlap/, + ); + assert.throws( + () => + validateMcsFleet({ + ...fleet, + members: fleet.members.map((m, i) => + i === 1 ? { ...m, cidr: "0.0.0.0/0" } : m, + ), + }), + /private VPC/, + ); +}); +test("mesh leaves existing Ohio-West connection and route ownership untouched", () => { + const props = { env, fleet, member: fleet.members[0] }; + const peering = Template.fromStack( + new McsPeeringsStack(new App(), "Peerings", props), + ); + peering.resourceCountIs("AWS::EC2::VPCPeeringConnection", 2); + const routes = Template.fromStack( + new McsRoutesStack(new App(), "Routes", props), + ); + routes.resourceCountIs("AWS::EC2::Route", 4); + const values = Object.values(routes.findResources("AWS::EC2::Route")); + assert.deepEqual( + new Set(values.map((v) => v.Properties.DestinationCidrBlock)), + new Set(["10.44.0.0/16", "10.45.0.0/16"]), + ); + assert.throws( + () => + new McsRoutesStack(new App(), "MissingId", { + ...props, + fleet: { + ...fleet, + links: fleet.links.map((l) => ({ ...l, peeringId: undefined })), + }, + }), + /populate peering IDs/, + ); +}); +test("member setup never owns cluster, node groups, or the running hub", () => { + const stack = new McsMemberStack(new App(), "Member", { + env, + fleet, + member: { + ...fleet.members[0], + hubApiService: { name: "karmada-apiserver", namespace: "karmada-system" }, + }, + }); + const t = Template.fromStack(stack); + for (const type of [ + "AWS::EKS::Cluster", + "AWS::EKS::Nodegroup", + "AWS::EC2::Instance", + "AWS::IAM::Role", + "Custom::AWSCDK-EKS-HelmChart", + ]) + t.resourceCountIs(type, 0); + t.resourceCountIs("AWS::EC2::SecurityGroupIngress", 6); + const patches = Object.values( + t.findResources("Custom::AWSCDK-EKS-KubernetesPatch"), + ); + assert.equal(patches.length, 2); + assert.ok( + JSON.stringify(patches).includes("AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS"), + ); + assert.ok( + JSON.stringify(patches).includes("10.43.0.0/16,10.44.0.0/16,10.45.0.0/16"), + ); + for (const value of Object.values(t.toJSON().Resources) as { + Type: string; + DeletionPolicy?: string; + }[]) + if (value.Type !== "AWS::CDK::Metadata") + assert.equal(value.DeletionPolicy, "Retain"); + assert.equal(stack.terminationProtection, true); +});