diff --git a/tests/tailor-images/trident-vm-testimage/README.md b/tests/tailor-images/trident-vm-testimage/README.md index 62305dd36..e4dc5a3dd 100644 --- a/tests/tailor-images/trident-vm-testimage/README.md +++ b/tests/tailor-images/trident-vm-testimage/README.md @@ -6,10 +6,24 @@ image and adds Trident as well as its dependencies. It also adds openssh-server to allow for remote access. Finally, it contains duplicate set of partitions to setup the disk layout for A/B update. -Available images: +Two sets of images are available: - regular - with verity +- with UKI usr-verity for ACL-agent-driven A/B update testing + +For both, a set of corresponding update images is available. The ACL-agent +variant reuses the servicing-style VM image layout but additionally installs +`trident-acl-agent` so storm ACL-agent scenarios can drive a real in-guest +agent talking to `tridentd`. The base (qcow2) image installs the package but +leaves `trident-acl-agent.service` disabled -- the storm scenario delivers a +fake kubeconfig at runtime and enables/starts the service itself. Only the +update image enables `trident-acl-agent.service` by default, since after a +real A/B update boots into it there is no test harness left to `systemctl +enable --now` it. Neither image preseeds `/etc/trident/trident-acl-agent.conf` +-- the harness never writes that file at all; instead it delivers a fake +kubeconfig to `/var/lib/kubelet/kubeconfig` and supplies the Nebraska +endpoint/appId/track per-request via the update-request annotation. ## Additional Prerequisites @@ -26,6 +40,7 @@ To build the base image, run: | Regular | `make artifacts/trident-vm-grub-testimage.qcow2` | `artifacts/trident-vm-grub-testimage.qcow2` | | With verity `qcow2` | `make artifacts/trident-vm-grub-verity-testimage.qcow2` | `artifacts/trident-vm-grub-verity-testimage.qcow2` | | With verity fixed size `vhd` | `make artifacts/trident-vm-grub-verity-azure-testimage.vhd` | `artifacts/trident-vm-grub-verity-azure-testimage.vhd` | +| ACL-agent UKI usr-verity `qcow2` | `make artifacts/trident-vm-acl-agent-testimage.qcow2` | `artifacts/trident-vm-acl-agent-testimage.qcow2` | To build the update images, run: @@ -33,3 +48,4 @@ To build the update images, run: | ---------- | ------------ | ----------- | | Regular | `make artifacts/trident-vm-grub-testimage.cosi` | `artifacts/trident-vm-grub-testimage.cosi` | | With verity | `make artifacts/trident-vm-grub-verity-testimage.cosi` | `artifacts/trident-vm-grub-verity-testimage.cosi` | +| ACL-agent UKI usr-verity | `make artifacts/trident-vm-acl-agent-update-testimage.cosi` | `artifacts/trident-vm-acl-agent-update-testimage.cosi` | diff --git a/tests/tailor-images/trident-vm-testimage/base/files/trident-acl-agent-cert-install.service b/tests/tailor-images/trident-vm-testimage/base/files/trident-acl-agent-cert-install.service new file mode 100644 index 000000000..4033001c9 --- /dev/null +++ b/tests/tailor-images/trident-vm-testimage/base/files/trident-acl-agent-cert-install.service @@ -0,0 +1,28 @@ +[Unit] +Description=Install trident-acl-agent storm test CA cert into system trust store +# Test-harness-only plumbing: this file only exists when a storm test run +# has written an ephemeral CA cert onto the persistent, non-A/B +# /var/lib/trident-acl-agent partition (see prepareVmForAclAgent in +# tools/storm/aclagent/tests/update.go). It never exists on a real +# deployment. Skips as a no-op (not a failure) when absent, e.g. before the +# harness has run at all, or for run-rollback, which never uses Nebraska. +ConditionPathExists=/var/lib/trident-acl-agent/nebraska-ca.pem +Before=trident-acl-agent.service + +[Service] +Type=oneshot +RemainAfterExit=yes +# Re-derives the system trust store install every boot: /etc is part of +# this usr-verity image's plain A/B-swapped root filesystem (root-a/root-b, +# see storage-acl-agent.yaml -- no overlay involved), so neither step below +# carries over across a reboot into the other slot, unlike the source cert +# on /var/lib/trident-acl-agent. +ExecStart=/bin/sh -c ' \ + cp /var/lib/trident-acl-agent/nebraska-ca.pem /etc/pki/ca-trust/source/anchors/trident-acl-agent-storm-test-ca.pem && \ + update-ca-trust extract && \ + cp /etc/pki/ca-trust/source/anchors/trident-acl-agent-storm-test-ca.pem /etc/pki/tls/certs/trident-acl-agent-storm-test-ca.pem && \ + openssl rehash /etc/pki/tls/certs \ + ' + +[Install] +WantedBy=multi-user.target diff --git a/tests/tailor-images/trident-vm-testimage/base/files/trident-acl-agent-override.conf b/tests/tailor-images/trident-vm-testimage/base/files/trident-acl-agent-override.conf new file mode 100644 index 000000000..5aff37639 --- /dev/null +++ b/tests/tailor-images/trident-vm-testimage/base/files/trident-acl-agent-override.conf @@ -0,0 +1,14 @@ +[Unit] +Wants=trident-acl-agent-cert-install.service +After=trident-acl-agent-cert-install.service + +[Service] +ExecStart= +ExecStart=/usr/bin/trident-acl-agent --verbosity trace --network-verbosity warn +Environment=TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX=acl.azure.com +Environment=TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY=IMAGE_VERSION +Environment=TRIDENT_ACL_AGENT_ORCHESTRATION_MODE=annotations +Environment=TRIDENT_ACL_AGENT_ORCHESTRATION_COOKIE_ENABLED=true +Environment=TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH=/etc/os-release +Environment=TRIDENT_ACL_AGENT_CURRENT_VERSION_FALLBACK=always +Environment=TRIDENT_ACL_AGENT_KUBERNETES_VALIDATE_NODE_UUID=true diff --git a/tests/tailor-images/trident-vm-testimage/by-scenario/acl-agent-base.yaml b/tests/tailor-images/trident-vm-testimage/by-scenario/acl-agent-base.yaml new file mode 100644 index 000000000..bc4ee9fe0 --- /dev/null +++ b/tests/tailor-images/trident-vm-testimage/by-scenario/acl-agent-base.yaml @@ -0,0 +1,37 @@ +# Base (qcow2) counterpart to acl-agent.yaml. The real A/B reboot lands on a +# different root than the one the storm harness SSHes into pre-stage. /etc/trident +# and /var/lib/kubelet are persistent, part-labeled partitions shared by both +# roots (see storage-acl-agent.yaml), but the systemd enablement symlink that +# "systemctl enable --now" writes lives under the root filesystem itself, so a +# live enable on the old root has no effect once the new root is booted. +# trident-acl-agent.service is therefore only enabled by default on the +# cosi (update) counterpart; the qcow2 base image enables it live via SSH +# during initial deploy. +outputs: + $replace: + - format: qcow2 +config: + storage: + $include: ./include/storage-acl-agent.yaml + os: + hostname: trident-acl-agent-testimg + packages: + install: + $remove: + - trident-service + $append: + - trident-acl + additionalFiles: + - source: base/files/99-dhcp-eth0.network + destination: /etc/systemd/network/99-dhcp-eth0.network + - source: ../common/files/sudoers-wheel + destination: /etc/sudoers.d/wheel + - source: base/files/trident-acl-agent-override.conf + destination: /etc/systemd/system/trident-acl-agent.service.d/override.conf + - source: base/files/trident-acl-agent-cert-install.service + destination: /etc/systemd/system/trident-acl-agent-cert-install.service + services: + enable: + - kdump + - tridentd.socket + - trident-acl-agent-cert-install.service diff --git a/tests/tailor-images/trident-vm-testimage/by-scenario/acl-agent.yaml b/tests/tailor-images/trident-vm-testimage/by-scenario/acl-agent.yaml new file mode 100644 index 000000000..b00bb504a --- /dev/null +++ b/tests/tailor-images/trident-vm-testimage/by-scenario/acl-agent.yaml @@ -0,0 +1,29 @@ +outputs: + $replace: + - format: cosi +config: + storage: + $include: ./include/storage-acl-agent.yaml + os: + hostname: trident-acl-agent-testimg + packages: + install: + $remove: + - trident-service + $append: + - trident-acl + additionalFiles: + - source: base/files/99-dhcp-eth0.network + destination: /etc/systemd/network/99-dhcp-eth0.network + - source: ../common/files/sudoers-wheel + destination: /etc/sudoers.d/wheel + - source: base/files/trident-acl-agent-override.conf + destination: /etc/systemd/system/trident-acl-agent.service.d/override.conf + - source: base/files/trident-acl-agent-cert-install.service + destination: /etc/systemd/system/trident-acl-agent-cert-install.service + services: + enable: + - kdump + - tridentd.socket + - trident-acl-agent-cert-install.service + - trident-acl-agent.service diff --git a/tests/tailor-images/trident-vm-testimage/by-scenario/root-verity+usr-verity.yaml b/tests/tailor-images/trident-vm-testimage/by-scenario/root-verity+usr-verity+acl-agent+acl-agent-base.yaml similarity index 100% rename from tests/tailor-images/trident-vm-testimage/by-scenario/root-verity+usr-verity.yaml rename to tests/tailor-images/trident-vm-testimage/by-scenario/root-verity+usr-verity+acl-agent+acl-agent-base.yaml diff --git a/tests/tailor-images/trident-vm-testimage/image.yaml b/tests/tailor-images/trident-vm-testimage/image.yaml index 47d0a4add..e99dd1136 100644 --- a/tests/tailor-images/trident-vm-testimage/image.yaml +++ b/tests/tailor-images/trident-vm-testimage/image.yaml @@ -2,11 +2,11 @@ name: trident-vm-testimage matrix: arch: [amd64, arm64] - scenario: [grub, grub-base, grub-verity, grub-verity-base, root-verity, usr-verity, grub-verity-azure, grub-verity-azure-base] + scenario: [grub, grub-base, grub-verity, grub-verity-base, root-verity, usr-verity, acl-agent, acl-agent-base, grub-verity-azure, grub-verity-azure-base] selectors: exclude: - - { arch: arm64, scenario: [grub-base, grub-verity-base, root-verity, usr-verity, grub-verity-azure, grub-verity-azure-base] } + - { arch: arm64, scenario: [grub-base, grub-verity-base, root-verity, usr-verity, acl-agent, acl-agent-base, grub-verity-azure, grub-verity-azure-base] } base: ref: qemu_guest diff --git a/tests/tailor-images/trident-vm-testimage/include/storage-acl-agent.yaml b/tests/tailor-images/trident-vm-testimage/include/storage-acl-agent.yaml new file mode 100644 index 000000000..82632328f --- /dev/null +++ b/tests/tailor-images/trident-vm-testimage/include/storage-acl-agent.yaml @@ -0,0 +1,101 @@ +bootType: efi +disks: + - partitionTableType: gpt + partitions: + - id: esp + type: esp + label: esp + size: 512M + - id: boot-a + size: 256M + - id: boot-b + size: 256M + - id: root-a + size: 4G + - id: root-b + size: 4G + - id: usr-a + size: 1G + - id: usr-b + size: 1G + - id: usr-hash-a + size: 128M + - id: usr-hash-b + size: 128M + - id: trident + label: trident + size: 512M + - id: trident-acl-agent + label: trident-acl-agent + size: 128M + - id: kubelet + label: kubelet + size: 128M + - id: etc-trident + label: etc-trident + size: 128M + - id: home + label: home + size: 1G + - id: srv + label: srv + size: 128M +verity: + - id: usrverity + name: usr + dataDeviceId: usr-a + hashDeviceId: usr-hash-a + dataDeviceMountIdType: uuid + hashDeviceMountIdType: uuid +filesystems: + - deviceId: esp + type: fat32 + mountPoint: + idType: part-label + path: /boot/efi + options: umask=0077 + - deviceId: boot-a + type: ext4 + mountPoint: + idType: uuid + path: /boot + - deviceId: usrverity + type: ext4 + mountPoint: + path: /usr + options: defaults,ro + - deviceId: root-a + type: ext4 + mountPoint: + idType: uuid + path: / + - deviceId: trident + type: ext4 + mountPoint: + idType: part-label + path: /var/lib/trident + - deviceId: trident-acl-agent + type: ext4 + mountPoint: + idType: part-label + path: /var/lib/trident-acl-agent + - deviceId: kubelet + type: ext4 + mountPoint: + idType: part-label + path: /var/lib/kubelet + - deviceId: etc-trident + type: ext4 + mountPoint: + idType: part-label + path: /etc/trident + - deviceId: home + type: ext4 + mountPoint: + idType: part-label + path: /home + - deviceId: srv + type: ext4 + mountPoint: + idType: part-label + path: /srv