diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 29dcfe26..fb578586 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -99,7 +99,7 @@ jobs: env: REGISTRY_SERVICE: ${{ inputs.registry-service }} REGISTRY_URL: localhost:5000 - ZOT_HOST: localhost + ZOT_HOST: 127.0.0.1 ZOT_PORT: 8080 - name: Show disk usage before running the tests if: always() @@ -110,7 +110,7 @@ jobs: env: REGISTRY_SERVICE: ${{ inputs.registry-service }} REGISTRY_URL: localhost:5000 - ZOT_HOST: localhost + ZOT_HOST: 127.0.0.1 ZOT_PORT: 8080 - name: Show disk usage after running the tests if: always() diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index be7dfa12..e84b5f37 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -99,7 +99,7 @@ jobs: env: REGISTRY_SERVICE: ${{ inputs.registry-service }} REGISTRY_URL: localhost:5000 - ZOT_HOST: localhost + ZOT_HOST: 127.0.0.1 ZOT_PORT: 8080 - name: Show disk usage after running the tests if: always() diff --git a/Makefile b/Makefile index b1e2f7df..a4080ff1 100644 --- a/Makefile +++ b/Makefile @@ -44,9 +44,12 @@ TOOLS_D := $(HACK_D)/tools REGCLIENT := $(TOOLS_D)/bin/regctl REGCLIENT_VERSION := v0.5.1 SKOPEO = $(TOOLS_D)/bin/skopeo +CONTAINERD = $(TOOLS_D)/bin/containerd +CTR = $(TOOLS_D)/bin/ctr export SKOPEO_VERSION = 1.13.0 BATS = $(TOOLS_D)/bin/bats BATS_VERSION := v1.10.0 +CONTAINERD_VERSION := v2.3.3 # OCI registry ZOT := $(TOOLS_D)/bin/zot ZOT_VERSION := v2.1.8 @@ -136,7 +139,7 @@ go-test: go tool cover -html coverage.txt -o $(HACK_D)/coverage.html .PHONY: download-tools -download-tools: $(GOLANGCI_LINT) $(REGCLIENT) $(ZOT) $(BATS) $(UMOCI) $(SKOPEO) +download-tools: $(GOLANGCI_LINT) $(REGCLIENT) $(ZOT) $(BATS) $(UMOCI) $(SKOPEO) $(CONTAINERD) $(GOLANGCI_LINT): @[ -x $(GOLANGCI_LINT) ] || \ @@ -170,6 +173,15 @@ $(SKOPEO): cd $(TOP_LEVEL); \ rm -rf $$tmpdir; +$(CONTAINERD): + @set -e; mkdir -p "$(TOOLS_D)/bin"; \ + tmpdir=$$(mktemp -d); \ + $(call dlbin,$$tmpdir/containerd.tar.gz,https://github.com/containerd/containerd/releases/download/$(CONTAINERD_VERSION)/containerd-$(CONTAINERD_VERSION:v%=%)-linux-$(GOARCH).tar.gz); \ + tar -xzf $$tmpdir/containerd.tar.gz -C $$tmpdir; \ + cp $$tmpdir/bin/containerd $(CONTAINERD); \ + cp $$tmpdir/bin/ctr $(CTR); \ + rm -rf $$tmpdir; + $(BATS): mkdir -p $(TOOLS_D)/bin rm -rf bats-core diff --git a/install-build-deps.sh b/install-build-deps.sh index 5cbc2290..6873b165 100755 --- a/install-build-deps.sh +++ b/install-build-deps.sh @@ -4,10 +4,12 @@ set -o errexit installdeps_fedora() { sudo dnf install \ + kubernetes-cni \ jq \ lxc-devel \ libcap-devel \ - libacl-devel + libacl-devel \ + runc # skopeo deps sudo dnf install \ gpgme-devel \ @@ -24,6 +26,7 @@ installdeps_fedora() { COMMON_DEBS=( apache2-utils build-essential + containernetworking-plugins cryptsetup-bin curl erofsfuse @@ -48,6 +51,7 @@ COMMON_DEBS=( parallel pkg-config psmisc + runc shellcheck squashfs-tools squashfuse diff --git a/pkg/overlay/pack.go b/pkg/overlay/pack.go index 71a129a9..02f707a2 100644 --- a/pkg/overlay/pack.go +++ b/pkg/overlay/pack.go @@ -284,6 +284,11 @@ func generateBlob(layerType types.LayerType, contents string, ociDir string, sou if err != nil { return nil, "", "", err } + // LZ4/Zstd selected by mkfs.erofs is filesystem-internal compression; + // containerd suffixes describe an additional wrapper around the blob. + if layerType.Type == "erofs" { + mediaType = types.ContainerdErofsLayerMediaType + } } return blob, mediaType, rootHash, nil } @@ -684,6 +689,16 @@ func unpackOne(l ispec.Descriptor, ociDir string, extractDir string) error { return nil } + if l.MediaType == types.ContainerdErofsLayerMediaType { + fsi := stackerfs.New(fstypes.FilesystemType("erofs")) + if fsi == nil { + return errors.Errorf("failed to initialize erofs filesystem handler") + } + + return fsi.ExtractSingle( + path.Join(ociDir, "blobs", "sha256", l.Digest.Encoded()), extractDir) + } + if fsi := stackerfs.NewFromMediaType(l.MediaType); fsi != nil { return fsi.ExtractSingle( path.Join(ociDir, "blobs", "sha256", l.Digest.Encoded()), extractDir) diff --git a/pkg/types/layer_type.go b/pkg/types/layer_type.go index dbab430a..860a6965 100644 --- a/pkg/types/layer_type.go +++ b/pkg/types/layer_type.go @@ -14,6 +14,9 @@ import ( var ErrEmptyLayers = errors.New("empty layers") +// ContainerdErofsLayerMediaType identifies an unwrapped native EROFS blob. +const ContainerdErofsLayerMediaType = "application/vnd.erofs.layer.v1" + type LayerType struct { Type string Verity verity.VerityMetadata @@ -85,6 +88,8 @@ func NewLayerTypeManifest(manifest ispec.Manifest) (LayerType, error) { case erofs.GenerateErofsMediaType(erofs.LZ4Compression): fallthrough case erofs.GenerateErofsMediaType(erofs.ZstdCompression): + fallthrough + case ContainerdErofsLayerMediaType: return NewLayerType("erofs", verity.VerityMetadata(verityMetadataPresent)) case ispec.MediaTypeImageLayerGzip: fallthrough diff --git a/pkg/types/types_test.go b/pkg/types/types_test.go index c741150f..4a04b15d 100644 --- a/pkg/types/types_test.go +++ b/pkg/types/types_test.go @@ -4,6 +4,8 @@ import ( "os" "reflect" "testing" + + ispec "github.com/opencontainers/image-spec/specs-go/v1" ) func parse(t *testing.T, content string) *Stackerfile { @@ -51,6 +53,20 @@ func TestDockerFrom(t *testing.T) { } } +func TestNewLayerTypeManifestContainerdErofs(t *testing.T) { + manifest := ispec.Manifest{ + Layers: []ispec.Descriptor{{MediaType: ContainerdErofsLayerMediaType}}, + } + + layerType, err := NewLayerTypeManifest(manifest) + if err != nil { + t.Fatalf("failed to parse containerd EROFS layer type: %s", err) + } + if layerType.Type != "erofs" { + t.Fatalf("expected erofs layer type, got %s", layerType.Type) + } +} + func TestDependencyOrder(t *testing.T) { content := `first: from: diff --git a/stacker.yaml b/stacker.yaml new file mode 100644 index 00000000..d4a483c2 --- /dev/null +++ b/stacker.yaml @@ -0,0 +1,6 @@ +test: + from: + type: oci + url: ${{BUSYBOX_OCI}} + run: | + echo hello > /hello diff --git a/test/containerd-erofs.bats b/test/containerd-erofs.bats new file mode 100644 index 00000000..9ec317ac --- /dev/null +++ b/test/containerd-erofs.bats @@ -0,0 +1,254 @@ +load helpers + +function setup() { + stacker_setup + CONTAINERD_ADDRESS="/tmp/stacker-containerd-${BATS_TEST_NUMBER}-$$.sock" +} + +function teardown() { + if [ -f "$TEST_TMPDIR/containerd.pid" ]; then + pid=$(cat "$TEST_TMPDIR/containerd.pid") + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + fi + rm -f "$CONTAINERD_ADDRESS" "$CONTAINERD_ADDRESS.ttrpc" + zot_teardown + cleanup +} + +function host_arch() { + case "$(uname -m)" in + x86_64) echo "amd64" ;; + aarch64) echo "arm64" ;; + *) + go env GOARCH + ;; + esac +} + +function write_containerd_config() { + local config_file="$1" + local hosts_config_path="$2" + local containerd_bin="$ROOT_DIR/hack/tools/bin/containerd" + local arch + arch=$(host_arch) + + "$containerd_bin" config default > "$config_file" + + # Keep the generated defaults and isolate only the state used by this test. + sed -i \ + -e "s|^root = .*|root = '$TEST_TMPDIR/containerd-root'|" \ + -e "s|^state = .*|state = '$TEST_TMPDIR/containerd-state'|" \ + -e "s|^\([[:space:]]*\)address = '/run/containerd/containerd.sock'$|\1address = '$CONTAINERD_ADDRESS'|" \ + -e "/io.containerd.service.v1.diff-service/,/^ \[plugins\./ s|^ default = .*| default = ['erofs', 'walking']|" \ + -e "/io.containerd.differ.v1.erofs/,/^ \[plugins\./ s|^ mkfs_options = .*| mkfs_options = ['--sort=none']|" \ + -e "/io.containerd.snapshotter.v1.erofs/,/^ \[plugins\./ s|^ root_path = .*| root_path = '$TEST_TMPDIR/containerd-erofs'|" \ + "$config_file" + + cat >> "$config_file" < "$hosts_dir/hosts.toml" < "$TEST_TMPDIR/containerd.log" 2>&1 & + echo $! > "$TEST_TMPDIR/containerd.pid" + + while [ "$n" -lt 30 ]; do + if "$ctr_bin" --address "$CONTAINERD_ADDRESS" plugins ls >/dev/null 2>&1; then + return 0 + fi + + n=$((n+1)) + sleep 1 + done + + echo "containerd failed to start" >&3 + cat "$TEST_TMPDIR/containerd.log" >&3 + return 1 +} + +function ensure_erofs_ready() { + run modinfo erofs + [ "$status" -eq 0 ] || skip "missing erofs kernel module" + + run modprobe erofs + [ "$status" -eq 0 ] || skip "unable to load erofs kernel module" + + run grep -Eq '^nodev[[:space:]]+erofs$|[[:space:]]erofs$' /proc/filesystems + [ "$status" -eq 0 ] || skip "erofs filesystem is not available" +} + +@test "stacker erofs image unpacks with containerd erofs snapshotter" { + require_privilege priv + + local containerd_bin="$ROOT_DIR/hack/tools/bin/containerd" + local ctr_bin="$ROOT_DIR/hack/tools/bin/ctr" + local manifest_digest + local image_ref + + [ -x "$containerd_bin" ] || skip "containerd test binary missing" + [ -x "$ctr_bin" ] || skip "ctr test binary missing" + + ensure_erofs_ready + + cat > stacker.yaml <<"EOF" +test: + from: + type: oci + url: ${{BUSYBOX_OCI}} + run: | + echo hello > /hello +EOF + + stacker build --layer-type=erofs --substitute BUSYBOX_OCI=${BUSYBOX_OCI} + + manifest_digest=$(jq -r '.manifests[0].digest' oci/index.json | cut -d: -f2) + mt="$(jq -r '.layers[0].mediaType' "oci/blobs/sha256/$manifest_digest")" + [ "$mt" = "application/vnd.erofs.layer.v1" ] + + run start_containerd + if [ "$status" -ne 0 ]; then + if grep -qE 'EROFS unsupported, please `modprobe erofs`|EROFS unsupported, please .*modprobe erofs' "$TEST_TMPDIR/containerd.log"; then + skip "erofs kernel support is unavailable" + fi + echo "containerd failed to start for unexpected reason" >&3 + cat "$TEST_TMPDIR/containerd.log" >&3 + return 1 + fi + + run "$ctr_bin" --address "$CONTAINERD_ADDRESS" plugins ls + [ "$status" -eq 0 ] + echo "$output" | grep -E "io\.containerd\.snapshotter\.v1\s+erofs\s+.*\s+ok" + echo "$output" | grep -E "io\.containerd\.differ\.v1\s+erofs\s+.*\s+ok" + + tar -C oci -cf "$TEST_TMPDIR/stacker-erofs.oci.tar" . + + run "$ctr_bin" --address "$CONTAINERD_ADDRESS" images import "$TEST_TMPDIR/stacker-erofs.oci.tar" + [ "$status" -eq 0 ] + + run "$ctr_bin" --address "$CONTAINERD_ADDRESS" images ls -q + [ "$status" -eq 0 ] + image_ref=$(echo "$output" | grep test-erofs | head -n1) + [ -n "$image_ref" ] + + run "$ctr_bin" --address "$CONTAINERD_ADDRESS" images unpack --snapshotter erofs "$image_ref" + [ "$status" -eq 0 ] + + run find "$TEST_TMPDIR/containerd-erofs" -type f -name layer.erofs + [ "$status" -eq 0 ] + [ -n "$output" ] +} + +@test "stacker erofs image published to zot runs through containerd mirror" { + require_privilege priv + + local containerd_bin="$ROOT_DIR/hack/tools/bin/containerd" + local ctr_bin="$ROOT_DIR/hack/tools/bin/ctr" + local mirror_registry="docker.io" + local mirror_repo="stacker-erofs-mirror-${BATS_TEST_NUMBER}" + local mirror_ref="$mirror_registry/library/$mirror_repo:latest" + + [ -x "$containerd_bin" ] || skip "containerd test binary missing" + [ -x "$ctr_bin" ] || skip "ctr test binary missing" + [ -n "${ZOT_HOST}${ZOT_PORT}" ] || skip "zot env not configured" + + ensure_erofs_ready + + zot_setup + + # Ensure Zot is actually up before proceeding + for i in $(seq 1 30); do + if curl -fsS "http://${ZOT_HOST}:${ZOT_PORT}/v2/" >/dev/null 2>&1; then + break + fi + sleep 1 + done + curl -fsS "http://${ZOT_HOST}:${ZOT_PORT}/v2/" >/dev/null || { + echo "zot health check failed" >&3 + return 1 + } + + cat > stacker.yaml <<"EOF" +test: + from: + type: oci + url: ${{BUSYBOX_OCI}} + run: | + echo hello-from-zot-mirror > /hello +EOF + + stacker build --layer-type=erofs --substitute BUSYBOX_OCI=${BUSYBOX_OCI} + stacker publish --skip-tls --url docker://${ZOT_HOST}:${ZOT_PORT} --image library/$mirror_repo --tag latest + + write_registry_mirror_hosts "$mirror_registry" + + run start_containerd_with_registry_config "$TEST_TMPDIR/certs.d" + if [ "$status" -ne 0 ]; then + if grep -qE 'EROFS unsupported, please `modprobe erofs`|EROFS unsupported, please .*modprobe erofs' "$TEST_TMPDIR/containerd.log"; then + skip "erofs kernel support is unavailable" + fi + echo "containerd failed to start for unexpected reason" >&3 + cat "$TEST_TMPDIR/containerd.log" >&3 + return 1 + fi + + run "$ctr_bin" --address "$CONTAINERD_ADDRESS" plugins ls + [ "$status" -eq 0 ] + echo "$output" | grep -E "io\.containerd\.snapshotter\.v1\s+erofs\s+.*\s+ok" + echo "$output" | grep -E "io\.containerd\.differ\.v1\s+erofs\s+.*\s+ok" + + # This image only exists in Zot; successful pull verifies mirror resolution. + run "$ctr_bin" --address "$CONTAINERD_ADDRESS" images pull "$mirror_ref" + [ "$status" -eq 0 ] + + run "$ctr_bin" --address "$CONTAINERD_ADDRESS" run --rm --snapshotter erofs "$mirror_ref" erofs-mirror-test sh -ec "cat /hello" + [ "$status" -eq 0 ] + echo "$output" | grep -q "hello-from-zot-mirror" + + run find "$TEST_TMPDIR/containerd-erofs" -type f -name layer.erofs + [ "$status" -eq 0 ] + [ -n "$output" ] +} diff --git a/test/helpers.bash b/test/helpers.bash index 2939a8a3..3a584ff1 100644 --- a/test/helpers.bash +++ b/test/helpers.bash @@ -310,40 +310,35 @@ function start_zot { echo "zot is running at pid $pid" cat "$TEST_TMPDIR/zot.log" - # wait until service is up - count=5 - up=0 + # wait until registry API is reachable; this avoids localhost IPv6/IPv4 races. + ready=0 + for i in $(seq 1 60); do + if [ ! -d /proc/$pid ]; then + echo "zot failed to start or died" + cat "$TEST_TMPDIR/zot.log" >&3 || true + exit 1 + fi - while [[ $count -gt 0 ]]; do - if [ ! -d /proc/$pid ]; then - echo "zot failed to start or died" - exit 1 - fi - up=1 - # check if correct port is open - if ! nc -v -z "${ZOT_HOST}" "${ZOT_PORT}"; then - echo "no response from host:${ZOT_HOST} port:${ZOT_PORT}" >&3 - sleep 1 - count=$((count - 1)) - continue - fi - echo "Got response from host:${ZOT_HOST} on port:${ZOT_PORT}" >&3 - if [[ -n $ZOT_USE_TLS ]]; then - echo "testing zot at https://$ZOT_HOST:$ZOT_PORT" - curl -v --cacert $BATS_SUITE_TMPDIR/ca.crt -u "iam:careful" -f https://$ZOT_HOST:$ZOT_PORT/v2/ || up=0 - else - echo "testing zot at http://$ZOT_HOST:$ZOT_PORT" - curl -v -f http://$ZOT_HOST:$ZOT_PORT/v2/ || up=0 - fi + if [[ -n $ZOT_USE_TLS ]]; then + if curl -fsS --connect-timeout 1 --cacert "$BATS_SUITE_TMPDIR/ca.crt" -u "iam:careful" "https://${ZOT_HOST}:${ZOT_PORT}/v2/" >/dev/null; then + ready=1 + break + fi + else + if curl -fsS --connect-timeout 1 "http://${ZOT_HOST}:${ZOT_PORT}/v2/" >/dev/null; then + ready=1 + break + fi + fi - if [ $up -eq 1 ]; then break; fi - sleep 1 - count=$((count - 1)) - done - if [ $up -eq 0 ]; then - echo "Timed out waiting for zot" - exit 1 - fi + sleep 0.5 + done + + if [ $ready -eq 0 ]; then + echo "Timed out waiting for zot at ${ZOT_HOST}:${ZOT_PORT}" >&3 + cat "$TEST_TMPDIR/zot.log" >&3 || true + exit 1 + fi echo "# zot is up" >&3 # setup a OCI client diff --git a/test/setup_suite.bash b/test/setup_suite.bash index cb02fd3e..0c705676 100644 --- a/test/setup_suite.bash +++ b/test/setup_suite.bash @@ -28,7 +28,7 @@ function write_certs { -CAkey ca.key \ -CAcreateserial \ -out server.cert \ - -extfile <(echo subjectAltName = DNS:localhost) + -extfile <(echo subjectAltName = DNS:localhost,IP:127.0.0.1) openssl req \ -newkey rsa:2048 \