From c5e86c161b276f3b8b9d1672eeb1f05f915d5ae5 Mon Sep 17 00:00:00 2001 From: Janis Horsts Date: Wed, 26 Aug 2026 12:21:46 +0100 Subject: [PATCH 1/9] Add --mount option to container k8s create --- Sources/ContainerK8s/Commands/K8sCreate.swift | 6 +++- .../Provisioners/LinuxNodeProvisioner.swift | 7 +++-- .../K8s/TestK8sRunSerial.swift | 29 +++++++++++++++++++ docs/command-reference.md | 3 +- 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/Sources/ContainerK8s/Commands/K8sCreate.swift b/Sources/ContainerK8s/Commands/K8sCreate.swift index 3f5d44155..740523c99 100644 --- a/Sources/ContainerK8s/Commands/K8sCreate.swift +++ b/Sources/ContainerK8s/Commands/K8sCreate.swift @@ -51,6 +51,9 @@ public struct K8sCreate: AsyncParsableCommand { @Option(help: "Node image reference (default: \(K8sHelper.nodeImage))") var nodeImage: String = K8sHelper.nodeImage + @Option(name: .customLong("mount"), help: "Add a mount to the container (format: type=<>,source=<>,target=<>,readonly)") + var mounts: [String] = [] + public func run() async throws { LoggingSystem.bootstrap { _ in StderrLogHandler() } let log = Logger(label: K8sHelper.pluginName) @@ -86,7 +89,8 @@ public struct K8sCreate: AsyncParsableCommand { registryScheme: registryFlags.scheme, maxConcurrentDownloads: imageFetchFlags.maxConcurrentDownloads, remove: remove, - fqdn: fqdn + fqdn: fqdn, + mounts: mounts ) progress.set(description: "Starting cluster") diff --git a/Sources/ContainerK8s/Provisioners/LinuxNodeProvisioner.swift b/Sources/ContainerK8s/Provisioners/LinuxNodeProvisioner.swift index c50f7d02b..405af3e53 100644 --- a/Sources/ContainerK8s/Provisioners/LinuxNodeProvisioner.swift +++ b/Sources/ContainerK8s/Provisioners/LinuxNodeProvisioner.swift @@ -32,6 +32,7 @@ public struct LinuxNodeProvisioner: NodeProvisioner { private let maxConcurrentDownloads: Int private let remove: Bool private let fqdn: String? + private let mounts: [String] public init( clusterName: String, @@ -42,7 +43,8 @@ public struct LinuxNodeProvisioner: NodeProvisioner { registryScheme: String = "https", maxConcurrentDownloads: Int = 3, remove: Bool = false, - fqdn: String? = nil + fqdn: String? = nil, + mounts: [String] = [] ) throws { guard !roles.isEmpty else { throw ContainerizationError(.invalidArgument, message: "LinuxNode roles must not be empty") @@ -63,6 +65,7 @@ public struct LinuxNodeProvisioner: NodeProvisioner { self.maxConcurrentDownloads = maxConcurrentDownloads self.remove = remove self.fqdn = fqdn + self.mounts = mounts } public func provision(name: String, log: Logger) async throws { @@ -90,7 +93,7 @@ public struct LinuxNodeProvisioner: NodeProvisioner { "\(ResourceLabelKeys.role)=\(roles.joined(separator: ","))", ], maskedPaths: [], - mounts: [], + mounts: mounts, name: name, networks: [], os: "linux", diff --git a/Tests/IntegrationTests/K8s/TestK8sRunSerial.swift b/Tests/IntegrationTests/K8s/TestK8sRunSerial.swift index 04f17e02d..777e01dcd 100644 --- a/Tests/IntegrationTests/K8s/TestK8sRunSerial.swift +++ b/Tests/IntegrationTests/K8s/TestK8sRunSerial.swift @@ -115,4 +115,33 @@ struct TestK8sRunSerial { #expect(server1 != server2) } } + + @Test func testCreateWithMount() async throws { + try await ContainerFixture.with { f in + let name = "k8s-\(f.testID)" + let testData = "hello k8s mount" + let hostFile = f.testDir.appending("testfile.txt") + try testData.write(toFile: hostFile.string, atomically: true, encoding: .utf8) + + f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) } + + try f.restoreWarmupImage(.kindestNodeV1_35_5) + print("[k8s-run] k8s create --name \(name) --mount type=virtiofs,source=\(f.testDir.string),target=/tmp/testmount,readonly") + let result = try f.run([ + "k8s", "create", + "--name", name, + "--mount", "type=virtiofs,source=\(f.testDir.string),target=/tmp/testmount,readonly", + ]) + print("[k8s-run] k8s create exit=\(result.status)") + if result.status != 0 { + print("[k8s-run] k8s create stderr: \(result.error)") + f.dumpNodeDiagnostics(node: name) + } + try result.check() + + let output = try f.doExec(name, cmd: ["cat", "/tmp/testmount/testfile.txt"]) + .trimmingCharacters(in: .whitespacesAndNewlines) + #expect(output == testData) + } + } } diff --git a/docs/command-reference.md b/docs/command-reference.md index d75ab5970..8cc98ba2d 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -1586,7 +1586,7 @@ Creates and starts a local Kubernetes cluster. Pulls the node image if needed, r **Usage** ```bash -container k8s create [--name ] [--node-image ] [--rm] [] [--debug] +container k8s create [--name ] [--node-image ] [--rm] [--mount ] [] [--debug] ``` **Options** @@ -1594,6 +1594,7 @@ container k8s create [--name ] [--node-image ] [--rm] [`: Cluster name (default: `k8s-dev`) * `--node-image `: Node image reference (default: `docker.io/kindest/node:v1.35.5`) * `--rm`: Remove the cluster container after it stops +* `--mount `: Add a mount to the container (format: type=<>,source=<>,target=<>,readonly) **Resource Options** From 56b3110353c268f6e5e5803da4e28e8ced2594c9 Mon Sep 17 00:00:00 2001 From: Janis Horsts Date: Wed, 26 Aug 2026 12:27:58 +0100 Subject: [PATCH 2/9] Args must be sorted --- Sources/ContainerK8s/Commands/K8sCreate.swift | 5 +++-- docs/command-reference.md | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Sources/ContainerK8s/Commands/K8sCreate.swift b/Sources/ContainerK8s/Commands/K8sCreate.swift index 740523c99..c97e55b24 100644 --- a/Sources/ContainerK8s/Commands/K8sCreate.swift +++ b/Sources/ContainerK8s/Commands/K8sCreate.swift @@ -36,6 +36,9 @@ public struct K8sCreate: AsyncParsableCommand { @Option(name: .long, help: "Cluster name (default: \(K8sHelper.defaultName))") var name: String = K8sHelper.defaultName + @Option(name: .customLong("mount"), help: "Add a mount to the container (format: type=<>,source=<>,target=<>,readonly)") + var mounts: [String] = [] + @Flag(name: [.customLong("rm"), .long], help: "Remove the cluster container after it stops") var remove: Bool = false @@ -51,8 +54,6 @@ public struct K8sCreate: AsyncParsableCommand { @Option(help: "Node image reference (default: \(K8sHelper.nodeImage))") var nodeImage: String = K8sHelper.nodeImage - @Option(name: .customLong("mount"), help: "Add a mount to the container (format: type=<>,source=<>,target=<>,readonly)") - var mounts: [String] = [] public func run() async throws { LoggingSystem.bootstrap { _ in StderrLogHandler() } diff --git a/docs/command-reference.md b/docs/command-reference.md index 8cc98ba2d..341cbff3c 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -1,7 +1,7 @@ # Container CLI Command Reference > [!IMPORTANT] -> This file contains documentation for the CURRENT BRANCH. To find documentation for official releases, find the target release on the [Release Page](https://github.com/apple/container/releases) and click the tag corresponding to your release version. +> This file contains documentation for the CURRENT BRANCH. To find documentation for official releases, find the target release on the [Release Page](https://github.com/apple/container/releases) and click the tag corresponding to your release version. > > Example: [release 0.4.1 tag](https://github.com/apple/container/tree/0.4.1) @@ -246,7 +246,7 @@ container create [] [ ...] * `--read-only-path `: **Experimental.** Mark a path inside the container read-only, in addition to the runtime defaults (or `NONE` to clear prior values and the defaults) * `--rm, --remove`: Remove the container after it stops * `--rosetta`: Enable Rosetta in the container -* `--runtime`: Set the runtime handler for the container (default: container-runtime-linux) +* `--runtime`: Set the runtime handler for the container (default: container-runtime-linux) * `--ssh`: Forward SSH agent socket to container * `--shm-size `: Size of `/dev/shm` (e.g. 64M, 1G) * `--tmpfs `: Add a tmpfs mount to the container at the given path @@ -1586,15 +1586,15 @@ Creates and starts a local Kubernetes cluster. Pulls the node image if needed, r **Usage** ```bash -container k8s create [--name ] [--node-image ] [--rm] [--mount ] [] [--debug] +container k8s create [--name ] [--node-image ] [--mount ] [--rm] [] [--debug] ``` **Options** +* `--mount `: Add a mount to the container (format: type=<>,source=<>,target=<>,readonly) * `--name `: Cluster name (default: `k8s-dev`) * `--node-image `: Node image reference (default: `docker.io/kindest/node:v1.35.5`) * `--rm`: Remove the cluster container after it stops -* `--mount `: Add a mount to the container (format: type=<>,source=<>,target=<>,readonly) **Resource Options** From 3668efa8e48811e4f92311220cbc3817aaf6699f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C4=81ris=20Vilks?= Date: Mon, 7 Sep 2026 11:00:46 +0300 Subject: [PATCH 3/9] fix(test): kindest/node boots systemd that mounts over the static /tmp thus making the test file unavailable; now mounting to /mnt --- Tests/IntegrationTests/K8s/TestK8sRunSerial.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tests/IntegrationTests/K8s/TestK8sRunSerial.swift b/Tests/IntegrationTests/K8s/TestK8sRunSerial.swift index 777e01dcd..66dcd551e 100644 --- a/Tests/IntegrationTests/K8s/TestK8sRunSerial.swift +++ b/Tests/IntegrationTests/K8s/TestK8sRunSerial.swift @@ -126,11 +126,11 @@ struct TestK8sRunSerial { f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) } try f.restoreWarmupImage(.kindestNodeV1_35_5) - print("[k8s-run] k8s create --name \(name) --mount type=virtiofs,source=\(f.testDir.string),target=/tmp/testmount,readonly") + print("[k8s-run] k8s create --name \(name) --mount type=virtiofs,source=\(f.testDir.string),target=/mnt/testmount,readonly") let result = try f.run([ "k8s", "create", "--name", name, - "--mount", "type=virtiofs,source=\(f.testDir.string),target=/tmp/testmount,readonly", + "--mount", "type=virtiofs,source=\(f.testDir.string),target=/mnt/testmount,readonly", ]) print("[k8s-run] k8s create exit=\(result.status)") if result.status != 0 { @@ -139,7 +139,7 @@ struct TestK8sRunSerial { } try result.check() - let output = try f.doExec(name, cmd: ["cat", "/tmp/testmount/testfile.txt"]) + let output = try f.doExec(name, cmd: ["cat", "/mnt/testmount/testfile.txt"]) .trimmingCharacters(in: .whitespacesAndNewlines) #expect(output == testData) } From 8ca5c80c380cdd925d87b497fb23eddb6b58843f Mon Sep 17 00:00:00 2001 From: Gerald Venzl Date: Wed, 9 Sep 2026 12:24:42 -0700 Subject: [PATCH 4/9] Update README as v1.0.0 has been reached (#2233) Co-authored-by: Eric Ernst --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 153e04573..e6d0cfdcb 100644 --- a/README.md +++ b/README.md @@ -99,4 +99,11 @@ Contributions to `container` are welcome and encouraged. Please see our [main co ## Project Status -The container project is currently under active development. Its stability, both for consuming the project as a Swift package and the `container` tool, is only guaranteed within patch versions, such as between 0.1.1 and 0.1.2. Minor version releases may include breaking changes until we reach a 1.0.0 release. +The container project is under active development. Its release versions are product versions, not semantic versions. + +The `container` CLI compatibility generally preserves backward compatibility within a major release (not breaking existing scripts), however, there may be the odd case where breaking compatibility may be necessary. **Note:** Features marked *experimental* (for example, the `k8s` subcommand) may change and do not guarantee backward compatibility. + +The `container-apiserver` XPC API compatibility preserves forward and backward compatibility within a major version. +Other non-public XPC helpers do not guarantee CLI or API compatibility across different versions. + +The `container` application data provides forward compatibility only, guaranteed within one major version. Upgrading to a newer major version may require a specific upgrade path. From d5c31e98a27554bdcf84cd82f871c753309e64d8 Mon Sep 17 00:00:00 2001 From: Dmitry Kovba Date: Thu, 10 Sep 2026 17:34:33 -0700 Subject: [PATCH 5/9] Fix a compilation warning (#2251) --- Tests/ContainerOSTests/DirectoryWatcherTest.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/Tests/ContainerOSTests/DirectoryWatcherTest.swift b/Tests/ContainerOSTests/DirectoryWatcherTest.swift index 0f4f671c7..82b87044a 100644 --- a/Tests/ContainerOSTests/DirectoryWatcherTest.swift +++ b/Tests/ContainerOSTests/DirectoryWatcherTest.swift @@ -16,7 +16,6 @@ import ContainerOS import ContainerizationError -import DNSServer import Foundation import SystemPackage import Testing From b61450b6e83ab0559217a4eee7485936e72ac395 Mon Sep 17 00:00:00 2001 From: Raj Date: Thu, 10 Sep 2026 19:00:07 -0700 Subject: [PATCH 6/9] Fix container egress loss after localhost DNS changes (#2256) --- .../Client/PacketFilter.swift | 124 +++++----- .../PacketFilterTest.swift | 228 +++++++++++++----- 2 files changed, 226 insertions(+), 126 deletions(-) diff --git a/Sources/Services/ContainerAPIService/Client/PacketFilter.swift b/Sources/Services/ContainerAPIService/Client/PacketFilter.swift index bd4035d2f..8bb1bb3fd 100644 --- a/Sources/Services/ContainerAPIService/Client/PacketFilter.swift +++ b/Sources/Services/ContainerAPIService/Client/PacketFilter.swift @@ -20,17 +20,26 @@ import DNSServer import Foundation import SystemPackage -public struct PacketFilter { - public static let anchor = "com.apple.container" +public struct PacketFilter: Sendable { + public static let anchor = "com.apple/container" public static let defaultConfigPath = FilePath("/etc/pf.conf") public static let defaultAnchorsPath = FilePath("/etc/pf.anchors") + private static let legacyAnchor = "com.apple.container" + private static let anchorFileName = "com.apple.container" + private let configPath: FilePath private let anchorsPath: FilePath + private let run: @Sendable ([String]) throws -> Int32 public init(configPath: FilePath = Self.defaultConfigPath, anchorsPath: FilePath = Self.defaultAnchorsPath) { + self.init(configPath: configPath, anchorsPath: anchorsPath, run: Self.runPFCTL) + } + + init(configPath: FilePath, anchorsPath: FilePath, run: @escaping @Sendable ([String]) throws -> Int32) { self.configPath = configPath self.anchorsPath = anchorsPath + self.run = run } public func createRedirectRule(from: IPAddress, to: IPAddress, domain: DNSName) throws { @@ -40,7 +49,7 @@ public struct PacketFilter { let fm: FileManager = FileManager.default - let anchorPath = self.anchorsPath.appending(Self.anchor) + let anchorPath = self.anchorsPath.appending(Self.anchorFileName) let inet: String switch from { @@ -52,9 +61,8 @@ public struct PacketFilter { var content = "" if fm.fileExists(atPath: anchorPath.string) { content = try String(contentsOfFile: anchorPath.string, encoding: .utf8) - } else { - try addAnchorToConfig() } + try updateConfig(removing: false) var lines = content.components(separatedBy: .newlines) if !content.contains(redirectRule) { @@ -71,7 +79,7 @@ public struct PacketFilter { let fm: FileManager = FileManager.default - let anchorPath = self.anchorsPath.appending(Self.anchor) + let anchorPath = self.anchorsPath.appending(Self.anchorFileName) let inet: String switch from { @@ -81,6 +89,7 @@ public struct PacketFilter { let redirectRule = "rdr \(inet) from any to \(from.description) -> \(to.description) # \(domain.pqdn)" guard fm.fileExists(atPath: anchorPath.string) else { + try updateConfig(removing: true) return } @@ -93,112 +102,89 @@ public struct PacketFilter { if removedLines == [""] { try fm.removeItem(atPath: anchorPath.string) - try removeAnchorFromConfig() + try updateConfig(removing: true) } else { try removedLines.joined(separator: "\n").write(toFile: anchorPath.string, atomically: true, encoding: .utf8) + try updateConfig(removing: false) } } - private func addAnchorToConfig() throws { + private func updateConfig(removing: Bool) throws { let fm: FileManager = FileManager.default - let anchorPath = self.anchorsPath.appending(Self.anchor) + let anchorPath = self.anchorsPath.appending(Self.anchorFileName) - /* PF requires strict ordering of anchors: - scrub-anchor, nat-anchor, rdr-anchor, dummynet-anchor, anchor, load anchor - */ - let anchorKeywords = ["scrub-anchor", "nat-anchor", "rdr-anchor", "dummynet-anchor", "anchor", "load anchor"] + let anchorKeywords = ["scrub-anchor", "nat-anchor", "rdr-anchor", "dummynet-anchor", "anchor"] let loadAnchorText = "load anchor \"\(Self.anchor)\" from \"\(anchorPath.string)\"" + let ownedLines = + anchorKeywords.map { "\($0) \"\(Self.legacyAnchor)\"" } + [ + "load anchor \"\(Self.legacyAnchor)\" from \"\(anchorPath.string)\"", + loadAnchorText, + ] var content: String = "" - var lines: [String] = [] if fm.fileExists(atPath: self.configPath.string) { content = try String(contentsOfFile: self.configPath.string, encoding: .utf8) } - lines = content.components(separatedBy: .newlines) - - for (i, keyword) in anchorKeywords[..<(anchorKeywords.endIndex - 1)].enumerated() { - let anchorText = "\(keyword) \"\(Self.anchor)\"" - - if content.contains(anchorText) { - continue + var lines = content.components(separatedBy: .newlines).filter { !ownedLines.contains($0) } + if !removing { + if lines.last != "" { + lines.append("") } - - let idx = lines.firstIndex { l in - anchorKeywords[i...].map { k in l.starts(with: k) }.contains(true) - } - lines.insert(anchorText, at: idx ?? lines.endIndex - 1) - } - - if !content.contains(loadAnchorText) { lines.insert(loadAnchorText, at: lines.endIndex - 1) } - do { - try lines.joined(separator: "\n").write(toFile: self.configPath.string, atomically: true, encoding: .utf8) - } catch { - throw ContainerizationError(.invalidState, message: "failed to write \"\(self.configPath.string)\"") - } - } - - private func removeAnchorFromConfig() throws { - let fm: FileManager = FileManager.default - - guard fm.fileExists(atPath: configPath.string) else { + let updatedContent = lines.joined(separator: "\n") + guard updatedContent != content else { return } - let content = try String(contentsOfFile: configPath.string, encoding: .utf8) - let lines = content.components(separatedBy: .newlines) - - let removedLines = lines.filter { l in !l.contains(Self.anchor) } - do { - try removedLines.joined(separator: "\n").write(toFile: configPath.string, atomically: true, encoding: .utf8) + try updatedContent.write(toFile: configPath.string, atomically: true, encoding: .utf8) } catch { throw ContainerizationError(.invalidState, message: "failed to write \"\(configPath.string)\"") } } public func reinitialize() throws { - let null = FileHandle.nullDevice - - let checkProcess = Foundation.Process() - var checkStatus: Int32 - checkProcess.executableURL = URL(fileURLWithPath: "/sbin/pfctl") - checkProcess.arguments = ["-n", "-f", configPath.string] - checkProcess.standardOutput = null - checkProcess.standardError = null + let anchorPath = self.anchorsPath.appending(Self.anchorFileName) + let path = FileManager.default.fileExists(atPath: anchorPath.string) ? anchorPath.string : "/dev/null" + let checkStatus: Int32 do { - try checkProcess.run() + checkStatus = try run(["-n", "-a", Self.anchor, "-f", path]) } catch { throw ContainerizationError(.internalError, message: "pfctl rule check exec failed: \"\(error)\"") } - checkProcess.waitUntilExit() - checkStatus = checkProcess.terminationStatus guard checkStatus == 0 else { - throw ContainerizationError(.internalError, message: "invalid pf config \"\(configPath.string)\"") + throw ContainerizationError(.internalError, message: "invalid pf config \"\(path)\"") } - let reloadProcess = Foundation.Process() - var reloadStatus: Int32 - - reloadProcess.executableURL = URL(fileURLWithPath: "/sbin/pfctl") - reloadProcess.arguments = ["-f", configPath.string] - reloadProcess.standardOutput = null - reloadProcess.standardError = null + try loadRules(anchor: Self.anchor, path: path) + try loadRules(anchor: Self.legacyAnchor, path: "/dev/null") + } + private func loadRules(anchor: String, path: String) throws { + let reloadStatus: Int32 do { - try reloadProcess.run() + reloadStatus = try run(["-a", anchor, "-f", path]) } catch { throw ContainerizationError(.internalError, message: "pfctl reload exec failed: \"\(error)\"") } - reloadProcess.waitUntilExit() - reloadStatus = reloadProcess.terminationStatus guard reloadStatus == 0 else { - throw ContainerizationError(.invalidState, message: "pfctl -f \"\(configPath.string)\" failed with status \(reloadStatus)") + throw ContainerizationError(.invalidState, message: "pfctl -a \"\(anchor)\" -f \"\(path)\" failed with status \(reloadStatus)") } } + + private static func runPFCTL(_ arguments: [String]) throws -> Int32 { + let process = Foundation.Process() + process.executableURL = URL(fileURLWithPath: "/sbin/pfctl") + process.arguments = arguments + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + try process.run() + process.waitUntilExit() + return process.terminationStatus + } } diff --git a/Tests/ContainerAPIClientTests/PacketFilterTest.swift b/Tests/ContainerAPIClientTests/PacketFilterTest.swift index 48cf049a1..587159787 100644 --- a/Tests/ContainerAPIClientTests/PacketFilterTest.swift +++ b/Tests/ContainerAPIClientTests/PacketFilterTest.swift @@ -18,6 +18,7 @@ import ContainerizationError import ContainerizationExtras import DNSServer import Foundation +import Synchronization import SystemPackage import Testing @@ -25,69 +26,182 @@ import Testing struct PacketFilterTest { @Test - func testRedirectRuleUpdate() async throws { - let fm = FileManager.default - let tempURL = try fm.url( - for: .itemReplacementDirectory, - in: .userDomainMask, - appropriateFor: .temporaryDirectory, - create: true - ) - let tempPath = FilePath(tempURL.path) - defer { try? FileManager.default.removeItem(at: tempURL) } - let configPath = tempPath.appending("pf.conf") - - let pf = PacketFilter(configPath: configPath, anchorsPath: tempPath) - let from1 = try! IPAddress("203.0.113.113") - let domain1 = try! DNSName("aaa.com") - let to = try! IPAddress("127.0.0.1") - try pf.createRedirectRule(from: from1, to: to, domain: domain1) - - let anchorPath = tempPath.appending("com.apple.container") - var actualAnchorText = try String(contentsOfFile: anchorPath.string, encoding: .utf8) - var expectedAnchorTest = """ - rdr inet from any to \(from1) -> \(to) # \(domain1.pqdn)\n - """ - - #expect(actualAnchorText == expectedAnchorTest) - - let from2 = try! IPAddress("172.31.72.1") - let domain2 = try! DNSName("bbb.com") - try pf.createRedirectRule(from: from2, to: to, domain: domain2) - - actualAnchorText = try String(contentsOfFile: anchorPath.string, encoding: .utf8) - expectedAnchorTest += """ - rdr inet from any to \(from2) -> \(to) # \(domain2.pqdn)\n - """ - #expect(actualAnchorText == expectedAnchorTest) - - let actualConfigText = try String(contentsOfFile: configPath.string, encoding: .utf8) - let expectedConfigText = try Regex( - #""" - scrub-anchor "([^"]+)" - nat-anchor "([^"]+)" - rdr-anchor "([^"]+)" - dummynet-anchor "([^"]+)" - anchor "([^"]+)" - load anchor "([^"]+)" from "[^"]+" - """# - ) + func testRedirectRuleLifecycle() throws { + try withTemporaryDirectory { tempPath in + let configPath = tempPath.appending("pf.conf") + let anchorPath = tempPath.appending("com.apple.container") + try String(Self.config.dropLast()).write(toFile: configPath.string, atomically: true, encoding: .utf8) + let commands = Mutex<[[String]]>([]) + let pf = PacketFilter(configPath: configPath, anchorsPath: tempPath) { arguments in + commands.withLock { $0.append(arguments) } + return 0 + } + let from1 = try IPAddress("203.0.113.113") + let from2 = try IPAddress("203.0.113.114") + let to = try IPAddress("127.0.0.1") + let domain1 = try DNSName("aaa.com") + let domain2 = try DNSName("bbb.com") + let rule1 = "rdr inet from any to \(from1) -> \(to) # \(domain1.pqdn)\n" + let rule2 = "rdr inet from any to \(from2) -> \(to) # \(domain2.pqdn)\n" + let configured = Self.config + "load anchor \"com.apple/container\" from \"\(anchorPath.string)\"\n" + let reloadCommands = [ + ["-n", "-a", "com.apple/container", "-f", anchorPath.string], + ["-a", "com.apple/container", "-f", anchorPath.string], + ["-a", "com.apple.container", "-f", "/dev/null"], + ] + + try pf.createRedirectRule(from: from1, to: to, domain: domain1) + try pf.createRedirectRule(from: from1, to: to, domain: domain1) + try pf.createRedirectRule(from: from2, to: to, domain: domain2) + try pf.reinitialize() + + #expect(try String(contentsOfFile: anchorPath.string, encoding: .utf8) == rule1 + rule2) + #expect(try String(contentsOfFile: configPath.string, encoding: .utf8) == configured) + #expect(commands.withLock { $0 } == reloadCommands) - #expect(actualConfigText.contains(expectedConfigText)) + try pf.removeRedirectRule(from: from1, to: to, domain: domain1) + try pf.reinitialize() + + #expect(try String(contentsOfFile: anchorPath.string, encoding: .utf8) == rule2) + #expect(try String(contentsOfFile: configPath.string, encoding: .utf8) == configured) + #expect(commands.withLock { $0 } == reloadCommands + reloadCommands) - try pf.removeRedirectRule(from: from1, to: to, domain: domain1) - try pf.removeRedirectRule(from: from2, to: to, domain: domain2) + try pf.removeRedirectRule(from: from2, to: to, domain: domain2) + try pf.reinitialize() - #expect(!fm.fileExists(atPath: anchorPath.string)) - let configText = try String(contentsOfFile: configPath.string, encoding: .utf8) - #expect(configText == "") + #expect(!FileManager.default.fileExists(atPath: anchorPath.string)) + #expect(try String(contentsOfFile: configPath.string, encoding: .utf8) == Self.config) + #expect(commands.withLock { $0 } == reloadCommands + reloadCommands + Self.emptyReloadCommands) + } } - @Test - func testPacketFilterReinitialize() async throws { - let pf = PacketFilter() - #expect(throws: ContainerizationError.self) { + @Test(arguments: [false, true]) + func testLegacyRulesMigration(deleting: Bool) throws { + try withTemporaryDirectory { tempPath in + let configPath = tempPath.appending("pf.conf") + let anchorPath = tempPath.appending("com.apple.container") + try Self.legacyConfig(anchorPath: anchorPath).write(toFile: configPath.string, atomically: true, encoding: .utf8) + let from = try IPAddress("203.0.113.113") + let to = try IPAddress("127.0.0.1") + let domain = try DNSName("aaa.com") + let rule = "rdr inet from any to \(from) -> \(to) # \(domain.pqdn)\n" + let retainedRule = "rdr inet from any to 203.0.113.114 -> 127.0.0.1 # bbb.com\n" + let originalRules = deleting ? rule + retainedRule : retainedRule + try originalRules.write(toFile: anchorPath.string, atomically: true, encoding: .utf8) + let commands = Mutex<[[String]]>([]) + let pf = PacketFilter(configPath: configPath, anchorsPath: tempPath) { arguments in + commands.withLock { $0.append(arguments) } + return 0 + } + + if deleting { + try pf.removeRedirectRule(from: from, to: to, domain: domain) + } else { + try pf.createRedirectRule(from: from, to: to, domain: domain) + } try pf.reinitialize() + + let expectedRules = deleting ? retainedRule : retainedRule + rule + let expectedConfig = Self.config + "load anchor \"com.apple/container\" from \"\(anchorPath.string)\"\n" + #expect(try String(contentsOfFile: anchorPath.string, encoding: .utf8) == expectedRules) + #expect(try String(contentsOfFile: configPath.string, encoding: .utf8) == expectedConfig) + #expect( + commands.withLock { $0 } == [ + ["-n", "-a", "com.apple/container", "-f", anchorPath.string], + ["-a", "com.apple/container", "-f", anchorPath.string], + ["-a", "com.apple.container", "-f", "/dev/null"], + ]) } } + + @Test(arguments: [false, true]) + func testLegacyLastRuleDeletion(missingFile: Bool) throws { + try withTemporaryDirectory { tempPath in + let configPath = tempPath.appending("pf.conf") + let anchorPath = tempPath.appending("com.apple.container") + try Self.legacyConfig(anchorPath: anchorPath).write(toFile: configPath.string, atomically: true, encoding: .utf8) + let from = try IPAddress("203.0.113.113") + let to = try IPAddress("127.0.0.1") + let domain = try DNSName("aaa.com") + if !missingFile { + let rule = "rdr inet from any to \(from) -> \(to) # \(domain.pqdn)\n" + try rule.write(toFile: anchorPath.string, atomically: true, encoding: .utf8) + } + let commands = Mutex<[[String]]>([]) + let pf = PacketFilter(configPath: configPath, anchorsPath: tempPath) { arguments in + commands.withLock { $0.append(arguments) } + return 0 + } + + try pf.removeRedirectRule(from: from, to: to, domain: domain) + try pf.reinitialize() + + #expect(!FileManager.default.fileExists(atPath: anchorPath.string)) + #expect(try String(contentsOfFile: configPath.string, encoding: .utf8) == Self.config) + #expect(commands.withLock { $0 } == Self.emptyReloadCommands) + } + } + + @Test(arguments: [0, 1, 2]) + func testReinitializeStopsOnFailure(failingCommand: Int) throws { + try withTemporaryDirectory { tempPath in + let commands = Mutex<[[String]]>([]) + let pf = PacketFilter(configPath: tempPath.appending("pf.conf"), anchorsPath: tempPath) { arguments in + commands.withLock { commands in + commands.append(arguments) + return commands.count - 1 == failingCommand ? 1 : 0 + } + } + + #expect { + try pf.reinitialize() + } throws: { error in + guard let error = error as? ContainerizationError else { + return false + } + return error.code == (failingCommand == 0 ? .internalError : .invalidState) + } + #expect(commands.withLock { $0 } == Array(Self.emptyReloadCommands.prefix(failingCommand + 1))) + } + } + + private static let config = """ + # Preserve com.apple.container configuration owned by other services. + scrub-anchor "com.apple/*" + nat-anchor "com.apple/*" + rdr-anchor "com.apple/*" + rdr-anchor "com.apple.container.other" + dummynet-anchor "com.apple/*" + anchor "com.apple/*" + load anchor "com.apple" from "/etc/pf.anchors/com.apple" + # load anchor "com.apple.container" from "/etc/pf.anchors/custom" + + """ + + private static let emptyReloadCommands = [ + ["-n", "-a", "com.apple/container", "-f", "/dev/null"], + ["-a", "com.apple/container", "-f", "/dev/null"], + ["-a", "com.apple.container", "-f", "/dev/null"], + ] + + private static func legacyConfig(anchorPath: FilePath) -> String { + var config = Self.config + for keyword in ["scrub-anchor", "nat-anchor", "rdr-anchor", "dummynet-anchor", "anchor"] { + let wildcard = "\(keyword) \"com.apple/*\"" + config = config.replacingOccurrences(of: "\n\(wildcard)\n", with: "\n\(keyword) \"com.apple.container\"\n\(wildcard)\n") + } + return config + "load anchor \"com.apple.container\" from \"\(anchorPath.string)\"\n" + } + + private func withTemporaryDirectory(_ body: (FilePath) throws -> Void) throws { + let fm = FileManager.default + let tempURL = try fm.url( + for: .itemReplacementDirectory, + in: .userDomainMask, + appropriateFor: .temporaryDirectory, + create: true + ) + defer { try? fm.removeItem(at: tempURL) } + try body(FilePath(tempURL.path)) + } } From 33ebc8ae67a06d148b3492dccfeb42ab00317470 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Fri, 11 Sep 2026 12:08:27 -0700 Subject: [PATCH 7/9] forwarder: close backend channel immediately if frontend closes (#2260) Signed-off-by: Eric Ernst --- Sources/SocketForwarder/ConnectHandler.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/SocketForwarder/ConnectHandler.swift b/Sources/SocketForwarder/ConnectHandler.swift index c55e3a538..a7589c242 100644 --- a/Sources/SocketForwarder/ConnectHandler.swift +++ b/Sources/SocketForwarder/ConnectHandler.swift @@ -62,7 +62,7 @@ extension ConnectHandler { case .success(let channel): guard context.channel.isActive else { self.log?.trace("backend - frontend channel closed, closing backend connection") - context.channel.close(promise: nil) + channel.close(promise: nil) return } self.log?.trace("backend - connected") From 55437109add247406b07644f9662f03ded52a5e7 Mon Sep 17 00:00:00 2001 From: Saiyam Pathak Date: Sat, 12 Sep 2026 04:05:24 +0530 Subject: [PATCH 8/9] docs: use an existing kindest/node tag in the k8s node image example (#2257) --- docs/kubernetes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 2817f4913..a3656e2fb 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -147,7 +147,7 @@ container k8s load-image --platform linux/amd64 my-app:latest By default, clusters use `kindest/node:v1.35.5`, a Kubernetes-in-Docker image optimized for local development. You can use a different node image when creating a cluster: ```bash -container k8s create --node-image docker.io/kindest/node:v1.34.4 +container k8s create --node-image docker.io/kindest/node:v1.34.11 ``` ## Cluster cleanup From 57f0b9392bbee1998e6c7f3f25db222fe1dcdd12 Mon Sep 17 00:00:00 2001 From: Kathryn Baldauf Date: Tue, 15 Sep 2026 09:36:17 -0700 Subject: [PATCH 9/9] K8s plugin: Support custom CNI manifest (#2254) Signed-off-by: Kathryn Baldauf --- Sources/ContainerK8s/Commands/K8sCreate.swift | 10 +++ .../Support/K8sHelper+Bootstrap.swift | 21 ++++-- Tests/K8sPluginTests/K8sCreateCNITests.swift | 64 +++++++++++++++++++ docs/command-reference.md | 8 ++- docs/kubernetes.md | 35 ++++++++++ 5 files changed, 130 insertions(+), 8 deletions(-) create mode 100644 Tests/K8sPluginTests/K8sCreateCNITests.swift diff --git a/Sources/ContainerK8s/Commands/K8sCreate.swift b/Sources/ContainerK8s/Commands/K8sCreate.swift index 3f5d44155..3432240e1 100644 --- a/Sources/ContainerK8s/Commands/K8sCreate.swift +++ b/Sources/ContainerK8s/Commands/K8sCreate.swift @@ -51,6 +51,9 @@ public struct K8sCreate: AsyncParsableCommand { @Option(help: "Node image reference (default: \(K8sHelper.nodeImage))") var nodeImage: String = K8sHelper.nodeImage + @Option(name: .long, help: "Optional path to a CNI manifest to apply.") + var cni: String? + public func run() async throws { LoggingSystem.bootstrap { _ in StderrLogHandler() } let log = Logger(label: K8sHelper.pluginName) @@ -59,6 +62,12 @@ public struct K8sCreate: AsyncParsableCommand { throw ContainerizationError(.invalidArgument, message: "cluster name \(name) is not a valid container ID") } + if let cni { + guard FileManager.default.fileExists(atPath: cni) else { + throw ContainerizationError(.invalidArgument, message: "CNI manifest not found at \(cni)") + } + } + let isTTY = isatty(FileHandle.standardError.fileDescriptor) == 1 let progressConfig = try ProgressConfig( showSpinner: isTTY, @@ -103,6 +112,7 @@ public struct K8sCreate: AsyncParsableCommand { try await K8sHelper.bootstrapControlPlane( nodeID: name, apiServerSANs: sans, advertiseAddress: vmIP, schedulable: provisioner.roles.contains(StandardRoles.worker), + cniManifestPath: cni, client: client, log: log) progress.set(description: "Waiting for cluster to be ready") diff --git a/Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift b/Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift index 964f9b72f..58e6bec33 100644 --- a/Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift +++ b/Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift @@ -35,7 +35,7 @@ extension K8sHelper { static func bootstrapControlPlane( nodeID: String, apiServerSANs: [String], advertiseAddress: String, - schedulable: Bool, client: ContainerClient, log: Logger + schedulable: Bool, cniManifestPath: String? = nil, client: ContainerClient, log: Logger ) async throws { let configYAML = initConfigYAML(advertiseAddress: advertiseAddress, certSANs: apiServerSANs) var r = try await execCapture( @@ -73,11 +73,9 @@ extension K8sHelper { arguments: ["taint", "nodes", "--all", "node-role.kubernetes.io/control-plane-"]) } - log.info("Applying kindnet CNI", metadata: ["node": "\(nodeID)"]) - let manifest = try await loadKindnetManifest(log: log) - let apply = - "cat > /tmp/kindnet.yaml <<'EOF'\n\(manifest)\nEOF\n" - + "\(kubeconfigEnv) kubectl apply -f /tmp/kindnet.yaml" + log.info("Applying CNI manifest", metadata: ["node": "\(nodeID)"]) + let manifest = try await loadCNIManifest(path: cniManifestPath, log: log) + let apply = "\(kubeconfigEnv) kubectl apply -f - <<'EOF'\n\(manifest)\nEOF" r = try await execCapture( containerId: nodeID, executable: "/bin/sh", arguments: ["-c", apply], client: client) @@ -103,6 +101,17 @@ extension K8sHelper { return (token: parts[tokenIdx + 1], caCertHash: parts[hashIdx + 1]) } + static func loadCNIManifest(path: String?, log: Logger) async throws -> String { + if let path { + do { + return try String(contentsOfFile: path, encoding: .utf8) + } catch { + throw ContainerizationError(.invalidArgument, message: "failed to read CNI manifest at \(path): \(error)") + } + } + return try await loadKindnetManifest(log: log) + } + private static func loadKindnetManifest(log: Logger) async throws -> String { let pluginLoader = try await Utility.createPluginLoader(log: log) guard let plugin = pluginLoader.findPlugin(forExecutable: CommandLine.executablePath), diff --git a/Tests/K8sPluginTests/K8sCreateCNITests.swift b/Tests/K8sPluginTests/K8sCreateCNITests.swift new file mode 100644 index 000000000..cafde5673 --- /dev/null +++ b/Tests/K8sPluginTests/K8sCreateCNITests.swift @@ -0,0 +1,64 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationError +import Foundation +import Logging +import Testing + +@testable import ContainerK8s + +// MARK: - K8sCreate flag parsing + +@Suite("K8sCreate --cni flag") +struct K8sCreateCNIFlagTests { + @Test func cniDefaultsToNilWhenNotProvided() throws { + let command = try K8sCreate.parse([]) + #expect(command.cni == nil) + } + + @Test func cniCapturesProvidedPath() throws { + let command = try K8sCreate.parse(["--cni", "/tmp/my-cni.yaml"]) + #expect(command.cni == "/tmp/my-cni.yaml") + } +} + +// MARK: - K8sHelper.loadCNIManifest + +@Suite("K8sHelper.loadCNIManifest") +struct LoadCNIManifestTests { + private let log = Logger(label: "test") + + @Test func customPathReturnsItsContents() async throws { + let contents = "kind: DaemonSet\nmetadata:\n name: my-custom-cni\n" + let dir = FileManager.default.temporaryDirectory + let url = dir.appendingPathComponent(UUID().uuidString + ".yaml") + try contents.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + + let result = try await K8sHelper.loadCNIManifest(path: url.path, log: log) + #expect(result == contents) + } + + @Test func missingPathThrowsInvalidArgument() async throws { + let missingPath = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString + "-does-not-exist.yaml").path + + await #expect(throws: ContainerizationError.self) { + _ = try await K8sHelper.loadCNIManifest(path: missingPath, log: log) + } + } +} diff --git a/docs/command-reference.md b/docs/command-reference.md index a99ac8528..9361036aa 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -1605,18 +1605,19 @@ container system property list --format json ### `container k8s create` -Creates and starts a local Kubernetes cluster. Pulls the node image if needed, runs `kubeadm init`, installs the kindnet CNI, and merges the cluster credentials into `~/.kube/config`. +Creates and starts a local Kubernetes cluster. Pulls the node image if needed, runs `kubeadm init`, installs a CNI (default: bundled kindnet), and merges the cluster credentials into `~/.kube/config`. **Usage** ```bash -container k8s create [--name ] [--node-image ] [--rm] [] [--debug] +container k8s create [--name ] [--node-image ] [--cni ] [--rm] [] [--debug] ``` **Options** * `--name `: Cluster name (default: `k8s-dev`) * `--node-image `: Node image reference (default: `docker.io/kindest/node:v1.35.5`) +* `--cni `: Optional path to a CNI manifest to apply. If not provided, the bundled kindnet CNI is used. * `--rm`: Remove the cluster container after it stops **Resource Options** @@ -1643,6 +1644,9 @@ container k8s create --name my-cluster --cpus 4 --memory 8g # create a cluster that removes itself when stopped container k8s create --name temp-cluster --rm + +# create a cluster using a custom CNI manifest instead of the bundled kindnet +container k8s create --cni ./my-cni.yaml ``` ### `container k8s start` diff --git a/docs/kubernetes.md b/docs/kubernetes.md index a3656e2fb..9e3e73a88 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -150,6 +150,41 @@ By default, clusters use `kindest/node:v1.35.5`, a Kubernetes-in-Docker image op container k8s create --node-image docker.io/kindest/node:v1.34.11 ``` +## Custom CNI + +By default, clusters install the bundled kindnet CNI for pod networking. Use `--cni` to apply a different CNI manifest instead: + +```bash +container k8s create --name my-cluster --cni ./my-cni.yaml +``` + +The manifest must be a plain Kubernetes YAML file (the same shape `kubectl apply -f` expects), not a Helm chart. + +### Example: Cilium + +Cilium is distributed as a Helm chart, so render a plain manifest from it first: + +```bash +helm repo add cilium https://helm.cilium.io/ +helm repo update +helm template cilium cilium/cilium --version 1.20.1 --namespace kube-system --set ipam.mode=kubernetes > cilium.yaml +``` + +`--set ipam.mode=kubernetes` avoids a CIDR conflict: the chart's default (`cluster-pool`, `10.0.0.0/8`) overlaps kubeadm's pod subnet and service CIDR on these clusters. + +Then create the cluster with that manifest: + +```bash +container k8s create --name cilium-demo --cni ./cilium.yaml +``` + +Verify Cilium came up: + +```bash +kubectl --context cilium-demo get pods -n kube-system +kubectl --context cilium-demo get nodes -o wide +``` + ## Cluster cleanup Remove a cluster and its data: