feat: provide RHEL-compatible 10 and Debian trixie builds

.github/workflows/build.yaml

@@ -28,7 +28,8 @@ jobs:
     outputs:
       build-type: ${{ steps.set-build-type.outputs.build-type }}
       date: ${{ steps.set-meta.outputs.date }}
-      linux-targets: ${{ steps.set-meta.outputs.linux-targets }}
+      # For PRs we can build for all possible targets but we use a reduced set for main/releases to keep costs/time down
+      linux-targets: ${{ github.event_name == 'pull_request' && steps.set-meta.outputs.pr-linux-targets || steps.set-meta.outputs.main-linux-targets }}
       version: ${{ steps.set-version.outputs.version }}
       oss-version: ${{ steps.set-oss-version.outputs.oss-version }}
     steps:
@@ -77,9 +78,13 @@ jobs:
         # Read from the file and remove newline characters: https://stackoverflow.com/a/64627966
         id: set-meta
         run: |
-          LINUX_TARGETS=$(cat "$JSON_FILE_NAME"|jq -c .linux_targets )
-          echo "linux-targets=$LINUX_TARGETS"
-          echo "linux-targets=$LINUX_TARGETS" >> $GITHUB_OUTPUT
+          PR_LINUX_TARGETS=$(cat "$JSON_FILE_NAME"|jq -c .linux_targets )
+          echo "pr-linux-targets=$PR_LINUX_TARGETS"
+          echo "pr-linux-targets=$PR_LINUX_TARGETS" >> $GITHUB_OUTPUT
+
+          MAIN_LINUX_TARGETS=$(cat "$JSON_FILE_NAME"|jq -c .release.linux_targets )
+          echo "main-linux-targets=$MAIN_LINUX_TARGETS"
+          echo "main-linux-targets=$MAIN_LINUX_TARGETS" >> $GITHUB_OUTPUT
         shell: bash
         env:
           JSON_FILE_NAME: build-config.json
@@ -113,9 +118,12 @@ jobs:
       version: ${{ needs.get-meta.outputs.version }}
       ref: ${{ github.ref }}
       image-base: ${{ matrix.image-base }}
-
       # Pick the Dockerfile to use for each image
       definition: ${{ (contains(matrix.image-base, 'debian') && 'Dockerfile.debian') || 'Dockerfile.ubi' }}
+      dockerhub-username: ${{ vars.DOCKERHUB_USERNAME }}
+    secrets:
+      dockerhub-token: ${{ secrets.DOCKERHUB_PUBLIC_READ_TOKEN }}
+
 
   # We want to copy the UBI image to ghcr.io/fluentdo/agent:version and
   # the distroless image to ghcr.io/fluentdo/agent:version-slim
@@ -209,6 +217,9 @@ jobs:
       ref: ${{ github.ref }}
       target-matrix: ${{ needs.get-meta.outputs.linux-targets }}
       nightly-build-info: ${{ needs.get-meta.outputs.date }}
+      dockerhub-username: ${{ vars.DOCKERHUB_USERNAME }}
+    secrets:
+      dockerhub-token: ${{ secrets.DOCKERHUB_PUBLIC_READ_TOKEN }}
 
   build-windows:
     # Only build Windows packages if we are not a pull request or have a label set
@@ -333,7 +344,10 @@ jobs:
             Version: ${{ needs.get-meta.outputs.version }}
             OSS Version: ${{ needs.get-meta.outputs.oss-version }}
 
-            Targets: ${{ needs.get-meta.outputs.linux-targets }}
+            Targets:
+              - Windows: 2022/2025
+              - Linux: ${{ needs.get-meta.outputs.linux-targets }}
+
             Images:
               - ${{ env.UBI_IMAGE_NAME }}:${{ env.TAG }}
               - ${{ env.DISTROLESS_IMAGE_NAME }}:${{ env.TAG }}

.github/workflows/call-build-containers.yaml

@@ -22,13 +22,21 @@ on:
         required: false
         type: string
         default: "Dockerfile.ubi"
+      dockerhub-username:
+        description: The Dockerhub username to use for authenticated pulls.
+        required: false
+        type: string
+        default: "fluentdo"
     secrets:
       cosign_private_key:
         description: The optional Cosign key to use for signing the images.
         required: false
       cosign_private_key_password:
         description: If the Cosign key requires a password then specify here, otherwise not required.
         required: false
+      dockerhub-token:
+        description: The Dockerhub token to use for authenticated pulls (not pushes).
+        required: true
     outputs:
       tag:
         description: The full image name and tag.
@@ -69,6 +77,12 @@ jobs:
       # - name: Set up Docker Buildx
       #   uses: docker/setup-buildx-action@v3
 
+      - name: Log in to docker.io for authorised pulls
+        uses: docker/login-action@v3
+        with:
+          username: ${{ inputs.dockerhub-username }}
+          password: ${{ secrets.dockerhub-token }}
+
       - name: Log in to the Container registry
         uses: docker/login-action@v3
         with:

.github/workflows/call-build-linux-packages.yaml

@@ -17,6 +17,22 @@ on:
         required: false
         type: string
         default: ""
+      dockerhub-username:
+        description: The Dockerhub username to use for authenticated pulls.
+        required: false
+        type: string
+        default: "fluentdo"
+
+    secrets:
+      gpg_private_key:
+        description: The optional GPG key to use for signing the packages.
+        required: false
+      gpg_private_key_password:
+        description: If the GPG key requires a password then specify here, otherwise not required.
+        required: false
+      dockerhub-token:
+        description: The Dockerhub token to use for authenticated pulls (not pushes).
+        required: true
 jobs:
   build-packages:
     name: agent - ${{ matrix.distro }} package build and upload
@@ -51,6 +67,12 @@ jobs:
           replace-with: "$1-$2"
           flags: "g"
 
+      - name: Log in to docker.io for authorised pulls
+        uses: docker/login-action@v3
+        with:
+          username: ${{ inputs.dockerhub-username }}
+          password: ${{ secrets.dockerhub-token }}
+
       - name: Log in to the GHCR registry
         uses: docker/login-action@v3
         with:

build-config.json

@@ -4,6 +4,8 @@
     "almalinux/8.arm64v8",
     "almalinux/9",
     "almalinux/9.arm64v8",
+    "almalinux/10",
+    "almalinux/10.arm64v8",
     "amazonlinux/2023",
     "amazonlinux/2023.arm64v8",
     "centos/6",
@@ -12,16 +14,24 @@
     "centos/8.arm64v8",
     "centos/9",
     "centos/9.arm64v8",
+    "centos/10",
+    "centos/10.arm64v8",
+    "mariner/2",
+    "mariner/2.arm64v8",
     "rockylinux/8",
     "rockylinux/8.arm64v8",
     "rockylinux/9",
     "rockylinux/9.arm64v8",
+    "rockylinux/10",
+    "rockylinux/10.arm64v8",
     "debian/buster",
     "debian/buster.arm64v8",
     "debian/bullseye",
     "debian/bullseye.arm64v8",
     "debian/bookworm",
     "debian/bookworm.arm64v8",
+    "debian/trixie",
+    "debian/trixie.arm64v8",
     "ubuntu/20.04",
     "ubuntu/20.04.arm64v8",
     "ubuntu/22.04",
@@ -31,6 +41,32 @@
     "suse/15",
     "suse/15.arm64v8"
   ],
+  "release": {
+    "linux_targets": [
+      "amazonlinux/2023",
+      "amazonlinux/2023.arm64v8",
+      "centos/6",
+      "centos/7",
+      "centos/10",
+      "centos/10.arm64v8",
+      "rockylinux/8",
+      "rockylinux/8.arm64v8",
+      "rockylinux/9",
+      "rockylinux/9.arm64v8",
+      "rockylinux/10",
+      "rockylinux/10.arm64v8",
+      "debian/bookworm",
+      "debian/bookworm.arm64v8",
+      "debian/trixie",
+      "debian/trixie.arm64v8",
+      "ubuntu/22.04",
+      "ubuntu/22.04.arm64v8",
+      "ubuntu/24.04",
+      "ubuntu/24.04.arm64v8",
+      "suse/15",
+      "suse/15.arm64v8"
+    ]
+  },
   "kind_versions": [
     "v1.31.12",
     "v1.32.8",

custom/packaging/distros/almalinux/Dockerfile

@@ -7,10 +7,10 @@ ARG BASE_BUILDER
 # Use buildkit to skip unused base images: DOCKER_BUILDKIT=1
 
 # Multiarch support
-FROM multiarch/qemu-user-static:x86_64-aarch64 as multiarch-aarch64
+FROM multiarch/qemu-user-static:x86_64-aarch64 AS multiarch-aarch64
 
 # almalinux/8 base image
-FROM almalinux:8 as almalinux-8-base
+FROM almalinux:8 AS almalinux-8-base
 
 # Add for the YAML development libraries
 RUN sed -i 's/enabled=0/enabled=1/g' /etc/yum.repos.d/almalinux-powertools.repo
@@ -25,7 +25,7 @@ RUN yum -y update && \
 
 # almalinux/8.arm64v8 base image
 # hadolint ignore=DL3029
-FROM --platform=arm64 almalinux:8 as almalinux-8.arm64v8-base
+FROM --platform=arm64 almalinux:8 AS almalinux-8.arm64v8-base
 
 COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static
 
@@ -44,7 +44,7 @@ RUN yum -y update && \
 ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3"
 ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS
 
-FROM almalinux:9 as almalinux-9-base
+FROM almalinux:9 AS almalinux-9-base
 
 # Add for the YAML development libraries
 RUN sed -i 's/enabled=0/enabled=1/g' /etc/yum.repos.d/almalinux-crb.repo
@@ -60,7 +60,42 @@ RUN yum -y update && \
 
 # almalinux/8.arm64v8 base image
 # hadolint ignore=DL3029
-FROM --platform=arm64 almalinux:9 as almalinux-9.arm64v8-base
+FROM --platform=arm64 almalinux:9 AS almalinux-9.arm64v8-base
+
+COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static
+
+# Add for the YAML development libraries
+RUN sed -i 's/enabled=0/enabled=1/g' /etc/yum.repos.d/almalinux-crb.repo
+
+# hadolint ignore=DL3033
+RUN yum -y update && \
+    yum install -y epel-release && \
+    yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
+    wget unzip systemd-devel wget flex bison \
+    postgresql-libs postgresql-devel postgresql-server postgresql \
+    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config && \
+    yum clean all
+
+# Need larger page size
+ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3"
+ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS
+
+FROM almalinux:10 AS almalinux-10-base
+
+# Add for the YAML development libraries
+RUN sed -i 's/enabled=0/enabled=1/g' /etc/yum.repos.d/almalinux-crb.repo
+
+# hadolint ignore=DL3033
+RUN yum -y update && \
+    yum install -y epel-release && \
+    yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
+    wget unzip systemd-devel wget flex bison \
+    postgresql-libs postgresql-devel postgresql-server postgresql \
+    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config && \
+    yum clean all
+
+# hadolint ignore=DL3029
+FROM --platform=arm64 almalinux:10 AS almalinux-10.arm64v8-base
 
 COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static
 
@@ -82,7 +117,7 @@ ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS
 
 # Common build for all distributions now
 # hadolint ignore=DL3006
-FROM $BASE_BUILDER as builder
+FROM $BASE_BUILDER AS builder
 
 ARG FLB_NIGHTLY_BUILD
 ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD

custom/packaging/distros/centos/Dockerfile

@@ -277,6 +277,79 @@ ENV FLB_PROCESSOR_DEDUP=$FLB_PROCESSOR_DEDUP
 ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3"
 ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS
 
+FROM quay.io/centos/centos:stream10 AS centos-10-base
+
+ENV CMAKE_HOME="/opt/cmake"
+ARG CMAKE_VERSION="3.31.6"
+ARG CMAKE_URL="https://github.com/Kitware/CMake/releases/download"
+
+# Add for the YAML development libraries
+# hadolint ignore=DL3033,DL3041
+RUN dnf -y install 'dnf-command(config-manager)' && dnf -y config-manager --set-enabled crb && \
+    dnf -y install epel-release && \
+    dnf -y install rpm-build ca-certificates gcc gcc-c++ make bash \
+    wget unzip systemd-devel wget flex bison \
+    postgresql-libs postgresql-devel postgresql-server postgresql \
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libcurl-devel pkgconf-pkg-config \
+    libyaml-devel zlib-devel libgit2-devel \
+    tar gzip && \
+    dnf clean all && \
+    mkdir -p "${CMAKE_HOME}" && \
+    cmake_download_url="${CMAKE_URL}/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz" && \
+    echo "Downloading CMake ${CMAKE_VERSION}: ${cmake_download_url} -> ${CMAKE_HOME}" && \
+    curl -jksSL "${cmake_download_url}" | tar -xzf - -C "${CMAKE_HOME}" --strip-components 1
+
+ENV PATH="${CMAKE_HOME}/bin:${PATH}"
+
+ARG FLB_OUT_PGSQL=On
+ENV FLB_OUT_PGSQL=$FLB_OUT_PGSQL
+ARG FLB_UNICODE_ENCODER=On
+ENV FLB_UNICODE_ENCODER=$FLB_UNICODE_ENCODER
+ARG FLB_KAFKA=On
+ENV FLB_KAFKA=$FLB_KAFKA
+ARG FLB_PROCESSOR_DEDUP=On
+ENV FLB_PROCESSOR_DEDUP=$FLB_PROCESSOR_DEDUP
+
+# hadolint ignore=DL3029
+FROM --platform=arm64 quay.io/centos/centos:stream10 AS centos-10.arm64v8-base
+
+COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static
+
+ENV CMAKE_HOME="/opt/cmake"
+ARG CMAKE_VERSION="3.31.6"
+ARG CMAKE_URL="https://github.com/Kitware/CMake/releases/download"
+
+# Add for the YAML development libraries
+# hadolint ignore=DL3033,DL3041
+RUN dnf -y install 'dnf-command(config-manager)' && dnf -y config-manager --set-enabled crb && \
+    dnf -y install epel-release && \
+    dnf -y install rpm-build ca-certificates gcc gcc-c++ make bash \
+    wget unzip systemd-devel wget flex bison \
+    postgresql-libs postgresql-devel postgresql-server postgresql \
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libcurl-devel pkgconf-pkg-config \
+    libyaml-devel zlib-devel libgit2-devel \
+    tar gzip && \
+    dnf clean all && \
+    mkdir -p "${CMAKE_HOME}" && \
+    cmake_download_url="${CMAKE_URL}/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz" && \
+    echo "Downloading CMake ${CMAKE_VERSION}: ${cmake_download_url} -> ${CMAKE_HOME}" && \
+    curl -jksSL "${cmake_download_url}" | tar -xzf - -C "${CMAKE_HOME}" --strip-components 1
+
+ENV PATH="${CMAKE_HOME}/bin:${PATH}"
+
+ARG FLB_OUT_PGSQL=On
+ENV FLB_OUT_PGSQL=$FLB_OUT_PGSQL
+ARG FLB_UNICODE_ENCODER=On
+ENV FLB_UNICODE_ENCODER=$FLB_UNICODE_ENCODER
+ARG FLB_KAFKA=On
+ENV FLB_KAFKA=$FLB_KAFKA
+ARG FLB_PROCESSOR_DEDUP=On
+ENV FLB_PROCESSOR_DEDUP=$FLB_PROCESSOR_DEDUP
+
+# Need larger page size
+ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3"
+ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS
+
 # Common build for all distributions now
 # hadolint ignore=DL3006
 FROM $BASE_BUILDER AS builder