Add --mount option to container k8s create

README.md

@@ -99,4 +99,11 @@ Contributions to `container` are welcome and encouraged. Please see our [main co
 
 ## Project Status
 
-The container project is currently under active development. Its stability, both for consuming the project as a Swift package and the `container` tool, is only guaranteed within patch versions, such as between 0.1.1 and 0.1.2. Minor version releases may include breaking changes until we reach a 1.0.0 release.
+The container project is under active development. Its release versions are product versions, not semantic versions.
+
+The `container` CLI compatibility generally preserves backward compatibility within a major release (not breaking existing scripts), however, there may be the odd case where breaking compatibility may be necessary. **Note:** Features marked *experimental* (for example, the `k8s` subcommand) may change and do not guarantee backward compatibility.
+
+The `container-apiserver` XPC API compatibility preserves forward and backward compatibility within a major version. 
+Other non-public XPC helpers do not guarantee CLI or API compatibility across different versions.
+
+The `container` application data provides forward compatibility only, guaranteed within one major version. Upgrading to a newer major version may require a specific upgrade path.

Sources/ContainerK8s/Commands/K8sCreate.swift

@@ -36,6 +36,9 @@ public struct K8sCreate: AsyncParsableCommand {
     @Option(name: .long, help: "Cluster name (default: \(K8sHelper.defaultName))")
     var name: String = K8sHelper.defaultName
 
+    @Option(name: .customLong("mount"), help: "Add a mount to the container (format: type=<>,source=<>,target=<>,readonly)")
+    var mounts: [String] = []
+
     @Flag(name: [.customLong("rm"), .long], help: "Remove the cluster container after it stops")
     var remove: Bool = false
 
@@ -51,6 +54,9 @@ public struct K8sCreate: AsyncParsableCommand {
     @Option(help: "Node image reference (default: \(K8sHelper.nodeImage))")
     var nodeImage: String = K8sHelper.nodeImage
 
+    @Option(name: .long, help: "Optional path to a CNI manifest to apply.")
+    var cni: String?
+
     public func run() async throws {
         LoggingSystem.bootstrap { _ in StderrLogHandler() }
         let log = Logger(label: K8sHelper.pluginName)
@@ -59,6 +65,12 @@ public struct K8sCreate: AsyncParsableCommand {
             throw ContainerizationError(.invalidArgument, message: "cluster name \(name) is not a valid container ID")
         }
 
+        if let cni {
+            guard FileManager.default.fileExists(atPath: cni) else {
+                throw ContainerizationError(.invalidArgument, message: "CNI manifest not found at \(cni)")
+            }
+        }
+
         let isTTY = isatty(FileHandle.standardError.fileDescriptor) == 1
         let progressConfig = try ProgressConfig(
             showSpinner: isTTY,
@@ -86,7 +98,8 @@ public struct K8sCreate: AsyncParsableCommand {
             registryScheme: registryFlags.scheme,
             maxConcurrentDownloads: imageFetchFlags.maxConcurrentDownloads,
             remove: remove,
-            fqdn: fqdn
+            fqdn: fqdn,
+            mounts: mounts
         )
 
         progress.set(description: "Starting cluster")
@@ -103,6 +116,7 @@ public struct K8sCreate: AsyncParsableCommand {
             try await K8sHelper.bootstrapControlPlane(
                 nodeID: name, apiServerSANs: sans, advertiseAddress: vmIP,
                 schedulable: provisioner.roles.contains(StandardRoles.worker),
+                cniManifestPath: cni,
                 client: client, log: log)
 
             progress.set(description: "Waiting for cluster to be ready")

Sources/ContainerK8s/Provisioners/LinuxNodeProvisioner.swift

@@ -32,6 +32,7 @@ public struct LinuxNodeProvisioner: NodeProvisioner {
     private let maxConcurrentDownloads: Int
     private let remove: Bool
     private let fqdn: String?
+    private let mounts: [String]
 
     public init(
         clusterName: String,
@@ -42,7 +43,8 @@ public struct LinuxNodeProvisioner: NodeProvisioner {
         registryScheme: String = "https",
         maxConcurrentDownloads: Int = 3,
         remove: Bool = false,
-        fqdn: String? = nil
+        fqdn: String? = nil,
+        mounts: [String] = []
     ) throws {
         guard !roles.isEmpty else {
             throw ContainerizationError(.invalidArgument, message: "LinuxNode roles must not be empty")
@@ -63,6 +65,7 @@ public struct LinuxNodeProvisioner: NodeProvisioner {
         self.maxConcurrentDownloads = maxConcurrentDownloads
         self.remove = remove
         self.fqdn = fqdn
+        self.mounts = mounts
     }
 
     public func provision(name: String, log: Logger) async throws {
@@ -90,7 +93,7 @@ public struct LinuxNodeProvisioner: NodeProvisioner {
                 "\(ResourceLabelKeys.role)=\(roles.joined(separator: ","))",
             ],
             maskedPaths: [],
-            mounts: [],
+            mounts: mounts,
             name: name,
             networks: [],
             os: "linux",

Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift

@@ -35,7 +35,7 @@ extension K8sHelper {
 
     static func bootstrapControlPlane(
         nodeID: String, apiServerSANs: [String], advertiseAddress: String,
-        schedulable: Bool, client: ContainerClient, log: Logger
+        schedulable: Bool, cniManifestPath: String? = nil, client: ContainerClient, log: Logger
     ) async throws {
         let configYAML = initConfigYAML(advertiseAddress: advertiseAddress, certSANs: apiServerSANs)
         var r = try await execCapture(
@@ -73,11 +73,9 @@ extension K8sHelper {
                 arguments: ["taint", "nodes", "--all", "node-role.kubernetes.io/control-plane-"])
         }
 
-        log.info("Applying kindnet CNI", metadata: ["node": "\(nodeID)"])
-        let manifest = try await loadKindnetManifest(log: log)
-        let apply =
-            "cat > /tmp/kindnet.yaml <<'EOF'\n\(manifest)\nEOF\n"
-            + "\(kubeconfigEnv) kubectl apply -f /tmp/kindnet.yaml"
+        log.info("Applying CNI manifest", metadata: ["node": "\(nodeID)"])
+        let manifest = try await loadCNIManifest(path: cniManifestPath, log: log)
+        let apply = "\(kubeconfigEnv) kubectl apply -f - <<'EOF'\n\(manifest)\nEOF"
         r = try await execCapture(
             containerId: nodeID, executable: "/bin/sh",
             arguments: ["-c", apply], client: client)
@@ -103,6 +101,17 @@ extension K8sHelper {
         return (token: parts[tokenIdx + 1], caCertHash: parts[hashIdx + 1])
     }
 
+    static func loadCNIManifest(path: String?, log: Logger) async throws -> String {
+        if let path {
+            do {
+                return try String(contentsOfFile: path, encoding: .utf8)
+            } catch {
+                throw ContainerizationError(.invalidArgument, message: "failed to read CNI manifest at \(path): \(error)")
+            }
+        }
+        return try await loadKindnetManifest(log: log)
+    }
+
     private static func loadKindnetManifest(log: Logger) async throws -> String {
         let pluginLoader = try await Utility.createPluginLoader(log: log)
         guard let plugin = pluginLoader.findPlugin(forExecutable: CommandLine.executablePath),

Sources/Services/ContainerAPIService/Client/PacketFilter.swift

@@ -20,17 +20,26 @@ import DNSServer
 import Foundation
 import SystemPackage
 
-public struct PacketFilter {
-    public static let anchor = "com.apple.container"
+public struct PacketFilter: Sendable {
+    public static let anchor = "com.apple/container"
     public static let defaultConfigPath = FilePath("/etc/pf.conf")
     public static let defaultAnchorsPath = FilePath("/etc/pf.anchors")
 
+    private static let legacyAnchor = "com.apple.container"
+    private static let anchorFileName = "com.apple.container"
+
     private let configPath: FilePath
     private let anchorsPath: FilePath
+    private let run: @Sendable ([String]) throws -> Int32
 
     public init(configPath: FilePath = Self.defaultConfigPath, anchorsPath: FilePath = Self.defaultAnchorsPath) {
+        self.init(configPath: configPath, anchorsPath: anchorsPath, run: Self.runPFCTL)
+    }
+
+    init(configPath: FilePath, anchorsPath: FilePath, run: @escaping @Sendable ([String]) throws -> Int32) {
         self.configPath = configPath
         self.anchorsPath = anchorsPath
+        self.run = run
     }
 
     public func createRedirectRule(from: IPAddress, to: IPAddress, domain: DNSName) throws {
@@ -40,7 +49,7 @@ public struct PacketFilter {
 
         let fm: FileManager = FileManager.default
 
-        let anchorPath = self.anchorsPath.appending(Self.anchor)
+        let anchorPath = self.anchorsPath.appending(Self.anchorFileName)
 
         let inet: String
         switch from {
@@ -52,9 +61,8 @@ public struct PacketFilter {
         var content = ""
         if fm.fileExists(atPath: anchorPath.string) {
             content = try String(contentsOfFile: anchorPath.string, encoding: .utf8)
-        } else {
-            try addAnchorToConfig()
         }
+        try updateConfig(removing: false)
 
         var lines = content.components(separatedBy: .newlines)
         if !content.contains(redirectRule) {
@@ -71,7 +79,7 @@ public struct PacketFilter {
 
         let fm: FileManager = FileManager.default
 
-        let anchorPath = self.anchorsPath.appending(Self.anchor)
+        let anchorPath = self.anchorsPath.appending(Self.anchorFileName)
 
         let inet: String
         switch from {
@@ -81,6 +89,7 @@ public struct PacketFilter {
         let redirectRule = "rdr \(inet) from any to \(from.description) -> \(to.description) # \(domain.pqdn)"
 
         guard fm.fileExists(atPath: anchorPath.string) else {
+            try updateConfig(removing: true)
             return
         }
 
@@ -93,112 +102,89 @@ public struct PacketFilter {
 
         if removedLines == [""] {
             try fm.removeItem(atPath: anchorPath.string)
-            try removeAnchorFromConfig()
+            try updateConfig(removing: true)
         } else {
             try removedLines.joined(separator: "\n").write(toFile: anchorPath.string, atomically: true, encoding: .utf8)
+            try updateConfig(removing: false)
         }
     }
 
-    private func addAnchorToConfig() throws {
+    private func updateConfig(removing: Bool) throws {
         let fm: FileManager = FileManager.default
 
-        let anchorPath = self.anchorsPath.appending(Self.anchor)
+        let anchorPath = self.anchorsPath.appending(Self.anchorFileName)
 
-        /* PF requires strict ordering of anchors:
-           scrub-anchor, nat-anchor, rdr-anchor, dummynet-anchor, anchor, load anchor
-         */
-        let anchorKeywords = ["scrub-anchor", "nat-anchor", "rdr-anchor", "dummynet-anchor", "anchor", "load anchor"]
+        let anchorKeywords = ["scrub-anchor", "nat-anchor", "rdr-anchor", "dummynet-anchor", "anchor"]
         let loadAnchorText = "load anchor \"\(Self.anchor)\" from \"\(anchorPath.string)\""
+        let ownedLines =
+            anchorKeywords.map { "\($0) \"\(Self.legacyAnchor)\"" } + [
+                "load anchor \"\(Self.legacyAnchor)\" from \"\(anchorPath.string)\"",
+                loadAnchorText,
+            ]
 
         var content: String = ""
-        var lines: [String] = []
         if fm.fileExists(atPath: self.configPath.string) {
             content = try String(contentsOfFile: self.configPath.string, encoding: .utf8)
         }
-        lines = content.components(separatedBy: .newlines)
-
-        for (i, keyword) in anchorKeywords[..<(anchorKeywords.endIndex - 1)].enumerated() {
-            let anchorText = "\(keyword) \"\(Self.anchor)\""
-
-            if content.contains(anchorText) {
-                continue
+        var lines = content.components(separatedBy: .newlines).filter { !ownedLines.contains($0) }
+        if !removing {
+            if lines.last != "" {
+                lines.append("")
             }
-
-            let idx = lines.firstIndex { l in
-                anchorKeywords[i...].map { k in l.starts(with: k) }.contains(true)
-            }
-            lines.insert(anchorText, at: idx ?? lines.endIndex - 1)
-        }
-
-        if !content.contains(loadAnchorText) {
             lines.insert(loadAnchorText, at: lines.endIndex - 1)
         }
 
-        do {
-            try lines.joined(separator: "\n").write(toFile: self.configPath.string, atomically: true, encoding: .utf8)
-        } catch {
-            throw ContainerizationError(.invalidState, message: "failed to write \"\(self.configPath.string)\"")
-        }
-    }
-
-    private func removeAnchorFromConfig() throws {
-        let fm: FileManager = FileManager.default
-
-        guard fm.fileExists(atPath: configPath.string) else {
+        let updatedContent = lines.joined(separator: "\n")
+        guard updatedContent != content else {
             return
         }
 
-        let content = try String(contentsOfFile: configPath.string, encoding: .utf8)
-        let lines = content.components(separatedBy: .newlines)
-
-        let removedLines = lines.filter { l in !l.contains(Self.anchor) }
-
         do {
-            try removedLines.joined(separator: "\n").write(toFile: configPath.string, atomically: true, encoding: .utf8)
+            try updatedContent.write(toFile: configPath.string, atomically: true, encoding: .utf8)
         } catch {
             throw ContainerizationError(.invalidState, message: "failed to write \"\(configPath.string)\"")
         }
     }
 
     public func reinitialize() throws {
-        let null = FileHandle.nullDevice
-
-        let checkProcess = Foundation.Process()
-        var checkStatus: Int32
-        checkProcess.executableURL = URL(fileURLWithPath: "/sbin/pfctl")
-        checkProcess.arguments = ["-n", "-f", configPath.string]
-        checkProcess.standardOutput = null
-        checkProcess.standardError = null
+        let anchorPath = self.anchorsPath.appending(Self.anchorFileName)
+        let path = FileManager.default.fileExists(atPath: anchorPath.string) ? anchorPath.string : "/dev/null"
 
+        let checkStatus: Int32
         do {
-            try checkProcess.run()
+            checkStatus = try run(["-n", "-a", Self.anchor, "-f", path])
         } catch {
             throw ContainerizationError(.internalError, message: "pfctl rule check exec failed: \"\(error)\"")
         }
 
-        checkProcess.waitUntilExit()
-        checkStatus = checkProcess.terminationStatus
         guard checkStatus == 0 else {
-            throw ContainerizationError(.internalError, message: "invalid pf config \"\(configPath.string)\"")
+            throw ContainerizationError(.internalError, message: "invalid pf config \"\(path)\"")
         }
 
-        let reloadProcess = Foundation.Process()
-        var reloadStatus: Int32
-
-        reloadProcess.executableURL = URL(fileURLWithPath: "/sbin/pfctl")
-        reloadProcess.arguments = ["-f", configPath.string]
-        reloadProcess.standardOutput = null
-        reloadProcess.standardError = null
+        try loadRules(anchor: Self.anchor, path: path)
+        try loadRules(anchor: Self.legacyAnchor, path: "/dev/null")
+    }
 
+    private func loadRules(anchor: String, path: String) throws {
+        let reloadStatus: Int32
         do {
-            try reloadProcess.run()
+            reloadStatus = try run(["-a", anchor, "-f", path])
         } catch {
             throw ContainerizationError(.internalError, message: "pfctl reload exec failed: \"\(error)\"")
         }
-        reloadProcess.waitUntilExit()
-        reloadStatus = reloadProcess.terminationStatus
         guard reloadStatus == 0 else {
-            throw ContainerizationError(.invalidState, message: "pfctl -f \"\(configPath.string)\" failed with status \(reloadStatus)")
+            throw ContainerizationError(.invalidState, message: "pfctl -a \"\(anchor)\" -f \"\(path)\" failed with status \(reloadStatus)")
         }
     }
+
+    private static func runPFCTL(_ arguments: [String]) throws -> Int32 {
+        let process = Foundation.Process()
+        process.executableURL = URL(fileURLWithPath: "/sbin/pfctl")
+        process.arguments = arguments
+        process.standardOutput = FileHandle.nullDevice
+        process.standardError = FileHandle.nullDevice
+        try process.run()
+        process.waitUntilExit()
+        return process.terminationStatus
+    }
 }

Sources/SocketForwarder/ConnectHandler.swift

@@ -62,7 +62,7 @@ extension ConnectHandler {
                 case .success(let channel):
                     guard context.channel.isActive else {
                         self.log?.trace("backend - frontend channel closed, closing backend connection")
-                        context.channel.close(promise: nil)
+                        channel.close(promise: nil)
                         return
                     }
                     self.log?.trace("backend - connected")