diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5c5403471b..0f00f10d5e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,75 +1,177 @@ name: Build Dyninst +# Reusable builder. Builds Dyninst's third-party libraries from source (see +# scripts/build-tpls.sh for why), then builds and installs Dyninst. +# +# Functional coverage lives in the rocprofiler-systems workflow; this job proves +# only that the tree configures, compiles and installs. +# +# The TPLs are built in a separate job because actions/cache skips its save step +# when a job fails: building them inline would mean a Dyninst build failure +# discards the TPL prefix and the next run pays for it again. + on: workflow_call: inputs: name: + description: Display name for this configuration required: true type: string - os: + image: + description: Container image to build in required: true type: string - extra-libs: + cc: required: false type: string - extra-cmake-flags: + default: gcc + cxx: required: false type: string - c-compiler: - required: true - type: string - cxx-compiler: - required: true + default: g++ + # RELWITHDEBINFO is RELEASE plus -g3 (see cmake/DyninstOptimization.cmake), + # so it covers the code generation of a RELEASE build, which is what + # rocprofiler-systems builds with, while leaving failures debuggable. + build-types: + description: JSON array of CMAKE_BUILD_TYPE values + required: false type: string - is-clang: + default: '["RELWITHDEBINFO"]' + container-options: required: false - type: boolean - default: false + type: string + default: '--shm-size=512m' + +permissions: + contents: read + +env: + TPL_PREFIX: ${{ github.workspace }}/.tpls + INSTALL_PREFIX: ${{ github.workspace }}/install jobs: + tpls: + name: ${{ inputs.name }} / third-party libs + runs-on: ubuntu-latest + container: + image: ${{ inputs.image }} + options: ${{ inputs.container-options }} + steps: + - uses: actions/checkout@v6 + + # The prefix bakes absolute paths (elfutils is built with an absolute + # RPATH), so the cache is only valid for an identical workspace path and + # image. Both are part of the key. + - name: Cache third-party libs + id: tpl-cache + uses: actions/cache@v6 + with: + path: ${{ env.TPL_PREFIX }} + key: tpls-${{ inputs.image }}-${{ hashFiles('scripts/tpl-versions.env', 'scripts/build-tpls.sh') }} + + - name: Build third-party libs + if: steps.tpl-cache.outputs.cache-hit != 'true' + run: bash scripts/build-tpls.sh --prefix "${TPL_PREFIX}" --jobs "$(nproc)" + build: - permissions: - packages: read + name: ${{ inputs.name }} (${{ matrix.build-type }}) + needs: tpls + runs-on: ubuntu-latest + container: + image: ${{ inputs.image }} + options: ${{ inputs.container-options }} + strategy: fail-fast: false matrix: - build-type: ['DEBUG', 'RELWITHDEBINFO', 'RELEASE'] - runs-on: ubuntu-latest - container: - image: ghcr.io/dyninst/amd64/${{ inputs.os }}:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.github_token }} - name: ${{ inputs.name }} (${{ matrix.build-type }}) + build-type: ${{ fromJSON(inputs.build-types) }} + + env: + CCACHE_DIR: ${{ github.workspace }}/.ccache + steps: - # Clang doesn't allow for multiple libomp installations - - name: Clean libomp install - if: ${{ inputs.is-clang }} - run: apt remove --purge -y "libomp*" - - - name: Install C compiler (${{ inputs.c-compiler }}) - run: | - apt update -qq - apt install -qq --no-install-recommends -y ${{ inputs.c-compiler }} + - uses: actions/checkout@v6 + + - name: Restore third-party libs + uses: actions/cache/restore@v6 + with: + path: ${{ env.TPL_PREFIX }} + key: tpls-${{ inputs.image }}-${{ hashFiles('scripts/tpl-versions.env', 'scripts/build-tpls.sh') }} + fail-on-cache-miss: true - # There is no apt package for clang++ - - name: Install ${{ inputs.cxx-compiler }} - if: ${{ !inputs.is-clang }} - run: apt install -qq --no-install-recommends -y ${{ inputs.cxx-compiler }} + - name: Restore ccache + uses: actions/cache@v6 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ inputs.image }}-${{ matrix.build-type }}-${{ github.sha }} + restore-keys: | + ccache-${{ inputs.image }}-${{ matrix.build-type }}- - - name: Install extra libs (${{ inputs.extra-libs }}) - if: ${{ inputs.extra-libs != '' }} - run: apt install -qq --no-install-recommends -y ${{ inputs.extra-libs }} + - name: Configure ccache + run: | + mkdir -p "${CCACHE_DIR}" + ccache --max-size=1G + ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime,pch_defines + ccache -z - - name: Configure Dyninst (${{ matrix.build-type }}) - shell: bash + - name: Configure run: | - cmake /dyninst/src \ - -DCMAKE_BUILD_TYPE="${{ matrix.build-type }}" \ - -DCMAKE_C_COMPILER="${{ inputs.c-compiler }}" \ - -DCMAKE_CXX_COMPILER="${{ inputs.cxx-compiler }}" \ - -DDYNINST_WARNINGS_AS_ERRORS=ON ${{ inputs.extra-cmake-flags }} + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + cmake --version + ${{ inputs.cxx }} --version + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \ + -DCMAKE_C_COMPILER=${{ inputs.cc }} \ + -DCMAKE_CXX_COMPILER=${{ inputs.cxx }} \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DDYNINST_WARNINGS_AS_ERRORS=ON \ + -DTBB_ROOT_DIR="${TPL_PREFIX}/tbb" \ + -DElfUtils_ROOT_DIR="${TPL_PREFIX}/elfutils" \ + -DLibIberty_ROOT_DIR="${TPL_PREFIX}/binutils" - - name: Build Dyninst + # A silent fallback to the distro's TBB/elfutils/libiberty would make this + # job green while testing the wrong dependency versions. LibIberty in + # particular does not reliably exclude system paths, so assert explicitly. + - name: Verify third-party libs resolved to the built prefix run: | - cmake --build . --parallel 2 + fail=0 + check() { + val=$(grep -E "^$1:" build/CMakeCache.txt | head -1 | cut -d= -f2-) + if [ -z "${val}" ]; then + echo "MISSING $1 is not set in CMakeCache.txt" + fail=1 + elif [ "${val#"${TPL_PREFIX}"}" != "${val}" ]; then + echo "ok $1 = ${val}" + else + echo "WRONG $1 = ${val}" + echo " expected a path under ${TPL_PREFIX}" + fail=1 + fi + } + check Elfutils_LIBRARIES + check LibIberty_LIBRARIES + check TBB_DIR + exit "${fail}" + + - name: Build + run: cmake --build build --parallel "$(nproc)" + + - name: Install + run: cmake --install build + + - name: ccache stats + if: always() + run: ccache -s + + - name: Upload CMake logs + if: failure() + uses: actions/upload-artifact@v7 + with: + name: cmake-logs-${{ inputs.name }}-${{ matrix.build-type }} + path: | + build/CMakeCache.txt + build/CMakeFiles/CMakeConfigureLog.yaml + build/CMakeFiles/*.log + if-no-files-found: ignore diff --git a/.github/workflows/cmake-formatting.yaml b/.github/workflows/cmake-formatting.yaml deleted file mode 100644 index 7507ed1277..0000000000 --- a/.github/workflows/cmake-formatting.yaml +++ /dev/null @@ -1,33 +0,0 @@ - -name: CMake Formatting - -on: - pull_request: - branches: [ master ] - paths: - - '**.cmake' - - '**CMakeLists.txt' - workflow_dispatch: - -jobs: - cmake-formatting: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y python3-pip - python3 -m pip install cmake-format - - name: cmake-format - run: | - set +e - cmake-format -i $(find . -type f | egrep 'CMakeLists.txt|\.cmake$') - if [ $(git diff | wc -l) -gt 0 ]; then - echo -e "\nError! CMake files not formatted." - echo -e "\nRun the following to fix:" - for f in $(git diff --name-only); do echo -e " cmake-format -i $f"; done - echo - exit 1 - fi diff --git a/.github/workflows/compiler-multibuild.yaml b/.github/workflows/compiler-multibuild.yaml deleted file mode 100644 index 9f2fa173d4..0000000000 --- a/.github/workflows/compiler-multibuild.yaml +++ /dev/null @@ -1,130 +0,0 @@ -# 1. Build with multiple versions of gcc and clang using the ubuntu-provided compiler -# 2. Using the latest version of each compiler, build against all supported C++ standards - -name: Compiler multibuild - -on: - schedule: - - cron: '0 3 * * 1' # 3AM on Monday - workflow_dispatch: - -jobs: - gcc-ubuntu-20_04: - strategy: - fail-fast: false - matrix: - version: [7, 8, 9, 10] - uses: ./.github/workflows/build.yaml - with: - name: gcc-${{ matrix.version }} - os: "ubuntu-20.04" - c-compiler: "gcc-${{ matrix.version }}" - cxx-compiler: "g++-${{ matrix.version }}" - - gcc-ubuntu-22_04: - strategy: - fail-fast: false - matrix: - version: [11, 12] - uses: ./.github/workflows/build.yaml - with: - name: gcc-${{ matrix.version }} - os: "ubuntu-22.04" - c-compiler: "gcc-${{ matrix.version }}" - cxx-compiler: "g++-${{ matrix.version }}" - - gcc-ubuntu-23_10: - strategy: - fail-fast: false - matrix: - version: [13] - uses: ./.github/workflows/build.yaml - with: - name: gcc-${{ matrix.version }} - os: "ubuntu-23.10" - c-compiler: "gcc-${{ matrix.version }}" - cxx-compiler: "g++-${{ matrix.version }}" - - - clang-ubuntu-20_04: - strategy: - fail-fast: false - matrix: - version: [7, 8, 9, 10, 11, 12] - uses: ./.github/workflows/build.yaml - with: - name: clang-${{ matrix.version }} - os: "ubuntu-20.04" - c-compiler: "clang-${{ matrix.version }}" - cxx-compiler: "clang++-${{ matrix.version }}" - is-clang: true - extra-libs: "libomp-${{ matrix.version }}-dev" - - clang-ubuntu-22_04: - strategy: - fail-fast: false - matrix: - version: [13, 14, 15] - uses: ./.github/workflows/build.yaml - with: - name: clang-${{ matrix.version }} - os: "ubuntu-22.04" - c-compiler: "clang-${{ matrix.version }}" - cxx-compiler: "clang++-${{ matrix.version }}" - is-clang: true - extra-libs: "libomp-${{ matrix.version }}-dev" - - clang-ubuntu-23_10: - strategy: - fail-fast: false - matrix: - version: [16, 17] - uses: ./.github/workflows/build.yaml - with: - name: clang-${{ matrix.version }} - os: "ubuntu-23.10" - c-compiler: "clang-${{ matrix.version }}" - cxx-compiler: "clang++-${{ matrix.version }}" - is-clang: true - extra-libs: "libomp-${{ matrix.version }}-dev" - - gcc-cxx-standards-11-17: - strategy: - fail-fast: false - matrix: - std: [11, 14, 17] - uses: ./.github/workflows/build.yaml - with: - name: gcc-cxx-${{ matrix.std }} - os: "ubuntu-22.04" - c-compiler: "gcc-12" - cxx-compiler: "g++-12" - extra-cmake-flags: "-DDYNINST_CXX_LANGUAGE_STANDARD=${{ matrix.std }}" - - gcc-cxx-standards-20-23: - strategy: - fail-fast: false - matrix: - std: [20, 23] - uses: ./.github/workflows/build.yaml - with: - name: gcc-cxx-${{ matrix.std }} - os: "ubuntu-23.10" - c-compiler: "gcc-13" - cxx-compiler: "g++-13" - extra-cmake-flags: "-DDYNINST_CXX_LANGUAGE_STANDARD=${{ matrix.std }}" - - clang-cxx-standards: - strategy: - fail-fast: false - matrix: - std: [11, 14, 17] # clang has a bug with 20+ and operator== reflexiveness - uses: ./.github/workflows/build.yaml - with: - name: cxx-${{ matrix.std }} - os: "ubuntu-22.04" - c-compiler: "clang-15" - cxx-compiler: "clang++-15" - is-clang: true - extra-cmake-flags: "-DDYNINST_CXX_LANGUAGE_STANDARD=${{ matrix.std }}" - extra-libs: "libomp-15-dev" diff --git a/.github/workflows/consumers.yaml b/.github/workflows/consumers.yaml deleted file mode 100644 index 3a21c3d023..0000000000 --- a/.github/workflows/consumers.yaml +++ /dev/null @@ -1,170 +0,0 @@ -# Build the latest versions of applications that consume Dyninst - -name: Build Consumers - -on: - schedule: - - cron: '0 3 * * 1' # Monday at 3AM - workflow_dispatch: - -jobs: - spack-build: - strategy: - fail-fast: false - matrix: - consumer: [ - "hpctoolkit@develop", - "must+stackwalker~backward~tsan" -# extrae+dyninst - not yet tested, may not support dyninst >10.0.0 -# omnitrace - Needs updated cmake -# timemory - Needs updated cmake - ] - runs-on: ubuntu-latest - steps: - - name: ${{ matrix.consumer }} - run: | - sudo apt update -qq - sudo apt install -y -qq --no-install-recommends build-essential gcc g++ gfortran m4 cmake autoconf python3 git unzip openmpi-bin libopenmpi-dev - git clone --depth=1 --branch=develop https://github.com/spack/spack - spack/bin/spack compiler find - spack/bin/spack external find --not-buildable cmake python git m4 openmpi gmake - spack/bin/spack install ${{ matrix.consumer }} ^dyninst@master - - systemtap: - permissions: - packages: read - strategy: - fail-fast: true - matrix: - os: ['ubuntu-23.10'] - runs-on: ubuntu-latest - container: - image: ghcr.io/dyninst/amd64/${{ matrix.os }}:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.github_token }} - name: systemtap ${{ matrix.os }} - steps: - - name: Install dependencies - run: | - apt update - apt install -y git python3 libjson-c-dev m4 autoconf - - name: Fetch systemtap - run: | - git clone --depth=1 https://sourceware.org/git/systemtap.git - - name: Make symlinks - run: | - ln -s /dyninst/install/include /usr/include/dyninst - ln -s /dyninst/install/lib /usr/lib64/dyninst - - name: Build systemtap - run: | - cd systemtap - autoreconf - mkdir build - cd build - ../configure --with-dyninst --without-python3-probes - make -j2 - - llnl-stat: - permissions: - packages: read - strategy: - fail-fast: true - matrix: - os: ['ubuntu-20.04'] - runs-on: ubuntu-latest - container: - image: ghcr.io/dyninst/amd64/${{ matrix.os }}:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.github_token }} - name: llnl-stat ${{ matrix.os }} - steps: - - name: Install dependencies - run: | - apt update - apt install -y nano git bison flex python3 build-essential dh-autoreconf wget libgcrypt20-dev libboost-program-options-dev libboost-regex-dev libboost-wave-dev libpython3-dev python3-distutils - - name: Install GraphLib - run: | - git clone --depth=1 https://github.com/LLNL/graphlib - cd graphlib - mkdir build - cd build - cmake .. - cmake --build . --parallel 2 - cmake --install . # /usr - - name: Install GraphViz - run: | - cd / - git clone --depth=1 https://gitlab.com/graphviz/graphviz.git - cd graphviz - ./autogen.sh - mkdir build - cd build - ../configure --without-qt --without-gts --without-doc --without-expat --without-ghostscript --without-gtkplus --without-libgd --without-pangocairo --without-popler --without-quartz --without-x - make -j2 - make install - - name: Install launchmon - run: | - cd / - git clone --depth=1 https://github.com/llnl/launchmon.git - cd launchmon - ./bootstrap - mkdir build - cd build - ../configure - make -j2 - make install - - name: Install MRNet - run: | - cd / - git clone --depth=1 https://github.com/dyninst/mrnet.git - cd mrnet - mkdir build - cd build - CC=gcc CXX=g++ ../configure --enable-shared - make -j2 - make install - - name: Install STAT - run: | - cd / - git clone --depth=1 https://github.com/llnl/stat.git - cd stat - ./bootstrap - mkdir build - cd build - ../configure --disable-gui --disable-examples --with-stackwalker=/dyninst/install --with-mrnet=/usr/local - make -j2 - tau: - permissions: - packages: read - strategy: - fail-fast: true - matrix: - os: ['ubuntu-20.04'] - runs-on: ubuntu-latest - container: - image: ghcr.io/dyninst/amd64/${{ matrix.os }}:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.github_token }} - name: TAU ${{ matrix.os }} - steps: - - name: Install dependencies - run: | - # TAU assumes Dyninst needs libdwarf instead of libdw. This has no real - # effect on Dyninst as we RPATH our deps. It just makes the manually-constructed - # link line in the TAU build work. - apt update - apt install -y git libdwarf1 - - name: Fix libdwarf - run: | - ln -s /usr/lib/x86_64-linux-gnu/libdwarf.so.1.0.0 /usr/lib/x86_64-linux-gnu/libdwarf.so - - name: Fetch TAU - run: | - git clone --depth=1 https://github.com/UO-OACISS/tau2 - - name: Build TAU - run: | - cd tau2 - ./configure -dyninst=/dyninst/install - make -j2 diff --git a/.github/workflows/dependency-version.yaml b/.github/workflows/dependency-version.yaml deleted file mode 100644 index bb2e3d4bb3..0000000000 --- a/.github/workflows/dependency-version.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# Ensure the minimum dependency versions found in the various CMake -# files match the expected values. This ensures we synchronize versions -# across containers and workflows. - -name: Check dependency versions - -on: - pull_request: - branches: [ master ] - paths: - - '**.cmake' - - '**CMakeLists.txt' - - 'docker/dependencies.versions' - workflow_dispatch: - -jobs: - check-version: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Version check - run: | - res=0 - current=$(awk 'match($0,/set\(_min_version (.+)\)/,a){print a[1]}' cmake/tpls/DyninstBoost.cmake) - expected=$(awk 'match($0,/boost:(.+)/,a){print a[1]}' docker/dependencies.versions) - if test "$current" != "$expected"; then - echo "Boost mismatch: Found $current, expected $expected" >/dev/stderr - res=1 - fi - - current=$(awk 'match($0,/set\(_min_version (.+)\)/,a){print a[1]}' cmake/tpls/DyninstTBB.cmake) - expected=$(awk 'match($0,/tbb:(.+)/,a){print a[1]}' docker/dependencies.versions) - if test "$current" != "$expected"; then - echo "TBB mismatch: Found $current, expected $expected" >/dev/stderr - res=1 - fi - - current=$(awk 'match($0,/set\(_min_version (.+)\)/,a){print a[1]}' cmake/tpls/DyninstElfUtils.cmake) - expected=$(awk 'match($0,/elfutils:(.+)/,a){print a[1]}' docker/dependencies.versions) - if test "$current" != "$expected"; then - echo "Elfutils mismatch: Found $current, expected $expected" >/dev/stderr - res=1 - fi - - current=$(awk 'match($0,/cmake_minimum_required\(VERSION (.+) FATAL_ERROR\)/,a){print a[1]}' CMakeLists.txt) - expected=$(awk 'match($0,/cmake:(.+)/,a){print a[1]}' docker/dependencies.versions) - if test "$current" != "$expected"; then - echo "CMake mismatch: Found $current, expected $expected" >/dev/stderr - res=1 - fi - - exit $res diff --git a/.github/workflows/dev-containers.yaml b/.github/workflows/dev-containers.yaml deleted file mode 100644 index d85fe5458e..0000000000 --- a/.github/workflows/dev-containers.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: Build and Deploy Development Containers - -on: - push: - branches: - - master - workflow_dispatch: - -jobs: - build: - permissions: - packages: write - strategy: - fail-fast: false - matrix: - os: ['ubuntu-20.04', 'ubuntu-22.04', 'ubuntu-23.04', 'ubuntu-23.10', 'ubuntu-24.04', 'fedora-37', 'fedora-38', 'fedora-39'] - runs-on: ubuntu-latest - name: Update dev containers - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: GHCR Login - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Pull base image - run: docker pull ghcr.io/dyninst/amd64/${{ matrix.os }}-base:latest - - - name: Build Dyninst Dev Container - run: | - cd docker/ - docker build --build-arg base=ghcr.io/dyninst/amd64/${{ matrix.os }}-base:latest \ - --build-arg build_jobs=2 \ - -f Dockerfile \ - -t ghcr.io/dyninst/amd64/${{ matrix.os }}:latest ../ - - - name: Deploy - run: docker push ghcr.io/dyninst/amd64/${{ matrix.os }}:latest diff --git a/.github/workflows/libabigail.yaml b/.github/workflows/libabigail.yaml deleted file mode 100644 index 36ab118611..0000000000 --- a/.github/workflows/libabigail.yaml +++ /dev/null @@ -1,95 +0,0 @@ -name: Libabigail ABI Checks -on: - pull_request: [] - -jobs: - get-release: - container: ghcr.io/dyninst/dyninst-ubuntu-20.04:v12.1.0 - runs-on: ubuntu-latest - steps: - - name: Upload Libs - uses: actions/upload-artifact@v2-preview - with: - name: release-libs - path: /opt/dyninst-env/install/dyninst/lib - - get-latest: - container: ghcr.io/dyninst/dyninst-ubuntu-20.04:latest - runs-on: ubuntu-latest - steps: - - name: Upload Libs - uses: actions/upload-artifact@v2-preview - with: - name: latest-libs - path: /opt/dyninst-env/install/dyninst/lib - - get-pr: - container: ghcr.io/dyninst/dyninst-ubuntu-20.04:latest - runs-on: ubuntu-latest - steps: - - name: Build Pull Request - uses: actions/checkout@v3 - - name: Build - run: | - rm -rf /code - cp -R $PWD /code - ls /code - cd /opt/dyninst-env - /bin/bash build.sh - - - name: Upload results - uses: actions/upload-artifact@v2-preview - with: - name: pr-libs - path: /opt/dyninst-env/install/dyninst/lib - - abi: - runs-on: ubuntu-latest - needs: [get-latest, get-release, get-pr] - strategy: - fail-fast: false - matrix: - - # Testing every paired library for release vs pr and main vs. pr - libs: ["libcommon.so", - "libdynC_API.so", - "libdynDwarf.so", - "libdynElf.so", - "libdyninstAPI_RT.so", - "libdyninstAPI.so", - "libinstructionAPI.so", - "libparseAPI.so", - "libpatchAPI.so", - "libpcontrol.so", - "libstackwalk.so", - "libsymLite.so", - "libsymtabAPI.so"] - - # Artifact pairs (named) for comparison) - artifacts: [["pr-libs", "latest-libs"], - ["pr-libs", "release-libs"]] - - steps: - - name: Download Previous Version - uses: actions/download-artifact@v2 - with: - name: ${{ matrix.artifacts[1] }} - path: previous/ - - - name: Download Pull Request Version - uses: actions/download-artifact@v2 - with: - name: ${{ matrix.artifacts[0] }} - path: current/ - - - name: Show Files - run: | - ls current/ - ls previous/ - - - name: Run Libabigail - uses: buildsi/libabigail-action@main - env: - lib: ${{ matrix.libs }} - with: - abidiff: previous/${{ env.lib }} current/${{ env.lib }} diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 0000000000..c6b4a5b5eb --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,61 @@ +name: Lint + +# Runs the repository's pre-commit hooks: cmake-format, shellcheck and +# actionlint. Contributors who ran `pre-commit install` can never fail this +# check, because it executes the same pinned hooks they do. +# +# Deliberately has no paths filter. The hooks already select their own files, +# and a filter here would only create ways for a change to skip the check. + +on: + pull_request: + branches: [dyninst_13] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + name: pre-commit + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + + # Tool versions are pinned in .pre-commit-config.yaml, not here. An + # unpinned formatter or linter can change its output and turn every PR red + # without any change to this repository. + - name: Install pre-commit + run: python3 -m pip install pre-commit==4.6.2 + + - name: Cache pre-commit environments + uses: actions/cache@v6 + with: + path: ~/.cache/pre-commit + key: pre-commit-3.12-${{ hashFiles('.pre-commit-config.yaml') }} + + - name: Run pre-commit + id: check + run: pre-commit run --all-files --show-diff-on-failure --color=always + + - name: How to fix + if: failure() && steps.check.outcome == 'failure' + run: | + echo "pre-commit reported problems. Anything printed as a diff above is" + echo "an automatic fix; shellcheck and actionlint findings need editing." + echo + echo "Reproduce locally with:" + echo " pip install pre-commit" + echo " pre-commit run --all-files" + echo + echo "Or run the hooks automatically on every commit:" + echo " pre-commit install" diff --git a/.github/workflows/pr-tests.yaml b/.github/workflows/pr-tests.yaml index ea8751f309..81ea37fefe 100644 --- a/.github/workflows/pr-tests.yaml +++ b/.github/workflows/pr-tests.yaml @@ -1,108 +1,48 @@ -# On each pull request, we build Dyninst, the test suite, the examples from -# dyninst/examples, and the external test from dyninst/external-tests -# -# The builds are carried out for each supported OS using the base containers -# at https://github.com/orgs/dyninst/packages +name: PR Tests -name: Pull Request Tests +# Goal 1: Dyninst builds and can instrument and run a program on its own. +# +# The images are the ROCm 7.2 primary entries of the rocprofiler-systems CI +# matrix, which are the environments this fork actually has to work in. They are +# listed here rather than fetched from ci-build-matrix.json because a standalone +# Dyninst build needs only the image name, and pinning the list keeps a change +# to the downstream matrix from silently altering what this workflow builds. +# +# Every entry is g++/gcc, which is what the reusable builder defaults to, so +# adding an amdclang++ row means explicitly passing cc/cxx. on: pull_request: - branches: - - master + branches: [dyninst_13] workflow_dispatch: -jobs: - gcc-build: - permissions: - packages: read - strategy: - fail-fast: false - matrix: - os: ['ubuntu-20.04', 'ubuntu-22.04', 'ubuntu-23.04', 'ubuntu-23.10', 'ubuntu-24.04', 'fedora-37', 'fedora-38', 'fedora-39'] - runs-on: ubuntu-latest - container: - image: ghcr.io/dyninst/amd64/${{ matrix.os }}-base:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.github_token }} - name: gcc on ${{ matrix.os }} - steps: - - name: Checkout Dyninst - uses: actions/checkout@v3 - with: - path: dyninst/src - - - name: Build Dyninst - run: | - ln -s $PWD/dyninst /dyninst - export DYNINST_C_FLAGS="-Werror" DYNINST_CXX_FLAGS="-Werror" - export DYNINST_C_COMPILER="gcc" DYNINST_CXX_COMPILER="g++" - bash /dyninst/src/docker/build.sh /dyninst/src 2 +permissions: + contents: read - - name: Checkout Test Suite - uses: actions/checkout@v3 - with: - repository: dyninst/testsuite - path: testsuite +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true - - name: Build testsuite - run: | - cd testsuite; mkdir build; cd build - cmake .. -DDyninst_DIR=/dyninst/install/lib/cmake/Dyninst - cmake --build . --parallel 2 - - - name: Checkout Examples - uses: actions/checkout@v3 - with: - repository: dyninst/examples - path: examples - - - name: Build examples - run: | - cd examples; mkdir build; cd build - cmake .. -DDyninst_DIR=/dyninst/install/lib/cmake/Dyninst - cmake --build . --parallel 2 - - - name: Checkout External Tests - uses: actions/checkout@v3 - with: - repository: dyninst/external-tests - path: external-tests - - - name: Build external tests - run: | - cd external-tests; mkdir build; cd build - cmake .. -DDyninst_DIR=/dyninst/install/lib/cmake/Dyninst - cmake --build . --parallel 2 - - name: Run tests - run: | - cd external-tests/build - ctest . - - clang-build: - permissions: - packages: read +jobs: + build: + name: ${{ matrix.name }} strategy: fail-fast: false matrix: - os: ['ubuntu-20.04', 'ubuntu-22.04', 'ubuntu-23.04', 'ubuntu-23.10', 'ubuntu-24.04'] - runs-on: ubuntu-latest - container: - image: ghcr.io/dyninst/amd64/${{ matrix.os }}-base:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.github_token }} - name: clang on ${{ matrix.os }} - steps: - - name: Checkout Dyninst - uses: actions/checkout@v3 - with: - path: dyninst/src - - - name: Build Dyninst - run: | - ln -s $PWD/dyninst /dyninst - export DYNINST_C_FLAGS="-Werror" DYNINST_CXX_FLAGS="-Werror" - export DYNINST_C_COMPILER="clang" DYNINST_CXX_COMPILER="clang++" - bash /dyninst/src/docker/build.sh /dyninst/src 2 + include: + - name: ubuntu-24.04 + image: dgaliffiamd/rocprofiler-systems:ci-rocm-7.2-ubuntu-24.04 + - name: ubuntu-22.04 + image: dgaliffiamd/rocprofiler-systems:ci-rocm-7.2-ubuntu-22.04 + - name: debian-12 + image: dgaliffiamd/rocprofiler-systems:ci-rocm-7.2-debian-12 + - name: rhel-8.10 + image: dgaliffiamd/rocprofiler-systems:ci-rocm-7.2-rhel-8.10 + - name: rhel-9 + image: dgaliffiamd/rocprofiler-systems:ci-rocm-7.2-rhel-9 + - name: rhel-10 + image: dgaliffiamd/rocprofiler-systems:ci-rocm-7.2-rhel-10 + uses: ./.github/workflows/build.yaml + with: + name: ${{ matrix.name }} + image: ${{ matrix.image }} diff --git a/.github/workflows/rocprofiler-systems.yaml b/.github/workflows/rocprofiler-systems.yaml new file mode 100644 index 0000000000..1c134dcf6e --- /dev/null +++ b/.github/workflows/rocprofiler-systems.yaml @@ -0,0 +1,415 @@ +name: rocprofiler-systems + +# Goal 2: rocprofiler-systems builds and its Dyninst-facing ctests pass when +# external/dyninst is this PR's head rather than the pinned submodule. +# +# This reproduces the upstream job in +# rocm-systems/.github/workflows/rocprofiler-systems-build-group.yml rather than +# calling it, because that workflow is not reusable from another repository. +# +# Advisory: the jobs are continue-on-error. rocm-systems develop moves +# independently of this repository, so a red result can mean upstream breakage +# rather than a regression in the pull request. Revisit once it has been stable +# for a while. + +on: + pull_request: + branches: [dyninst_13] + workflow_dispatch: + inputs: + scope: + description: Which upstream CI entries to run + type: choice + options: [single, full] + default: full + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + ROCM_SYSTEMS_REPO: ROCm/rocm-systems + ROCM_SYSTEMS_BRANCH: develop + +jobs: + # Reads the upstream matrix definition instead of copying it, so the set of + # images cannot drift from what rocprofiler-systems actually tests. + matrix: + name: resolve upstream matrix + runs-on: ubuntu-latest + outputs: + entries: ${{ steps.select.outputs.entries }} + sha: ${{ steps.resolve.outputs.sha }} + steps: + # Resolved once and reused by every build job, so a push to develop + # mid-run cannot leave the matrix entries testing different commits. + - name: Resolve rocm-systems branch to a commit + id: resolve + run: | + sha=$(git ls-remote "https://github.com/${ROCM_SYSTEMS_REPO}.git" \ + "refs/heads/${ROCM_SYSTEMS_BRANCH}" | cut -f1) + if [ -z "${sha}" ]; then + echo "Could not resolve ${ROCM_SYSTEMS_REPO}@${ROCM_SYSTEMS_BRANCH}" + exit 1 + fi + echo "sha=${sha}" >> "${GITHUB_OUTPUT}" + + - name: Select matrix entries + id: select + env: + # 'single' remains available through workflow_dispatch for bisecting a + # failure down to one image without paying for the other seven. + SCOPE: ${{ github.event.inputs.scope || 'full' }} + SHA: ${{ steps.resolve.outputs.sha }} + run: | + curl -fsSL -o matrix.json \ + "https://raw.githubusercontent.com/${ROCM_SYSTEMS_REPO}/${SHA}/projects/rocprofiler-systems/.github/ci-build-matrix.json" + + # kind drives the only real difference between the two upstream jobs: + # whether the Dyninst third-party libraries are built or taken from + # the distro. + if [ "${SCOPE}" = "full" ]; then + jq -c '[ + (.primary[] + | select(.image | test("ci-rocm-7\\.2-")) + | . + {kind: "primary"}), + (.system_deps[] + | select(.image | test("ci-rocm-7\\.2-ubuntu")) + | select(.compiler == "g++") + | . + {kind: "system_deps"}) + ]' matrix.json > entries.json + else + jq -c '[ + .primary[] + | select(.image | endswith("ci-rocm-7.2-ubuntu-24.04")) + | . + {kind: "primary"} + ]' matrix.json > entries.json + fi + + # An upstream rename would otherwise silently produce an empty matrix, + # which GitHub reports as a successful job. + count=$(jq 'length' entries.json) + if [ "${count}" -eq 0 ]; then + echo "No matrix entries matched. Upstream ci-build-matrix.json may have changed." + jq -r '.primary[].image, .system_deps[].image' matrix.json | sort -u + exit 1 + fi + echo "Selected ${count} entry/entries:" + jq -r '.[].name' entries.json + + echo "entries=$(jq -c '{include: .}' entries.json)" >> "${GITHUB_OUTPUT}" + + downstream: + name: ${{ matrix.name }} + needs: matrix + runs-on: ubuntu-latest + continue-on-error: true + container: + image: ${{ matrix.image }} + options: ${{ matrix.container_opts || '--shm-size=512m' }} + + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.entries) }} + + defaults: + run: + shell: bash + # Not under GITHUB_WORKSPACE; see "Move the checkout off the dyninst path". + working-directory: /rocm-systems/projects/rocprofiler-systems + + env: + CDASH_NAME: dyninst-${{ github.event.number || github.ref_name }}-${{ matrix.cdash_suffix }} + OMPI_ALLOW_RUN_AS_ROOT: 1 + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 + ROCPROFSYS_CI: 'ON' + ROCPROFSYS_MAX_THREADS: '64' + ROCPROFSYS_KEEP_TEST_OUTPUT: '0' + ROCPROFSYS_TMPDIR: "%env{PWD}%/testing-tmp" + CCACHE_DIR: ${{ github.workspace }}/.ccache + GIT_DISCOVERY_ACROSS_FILESYSTEM: 1 + CMAKE_C_COMPILER_LAUNCHER: ccache + CMAKE_CXX_COMPILER_LAUNCHER: ccache + + steps: + - name: Check out rocm-systems + uses: actions/checkout@v6 + with: + repository: ${{ env.ROCM_SYSTEMS_REPO }} + ref: ${{ needs.matrix.outputs.sha }} + path: rocm-systems + sparse-checkout: | + projects/rocprofiler-systems/ + .gitmodules + + # The whole point of this workflow. rocprofiler_systems_checkout_git_submodule() + # returns early when the submodule's CMakeLists.txt already exists, so + # populating the path here means CMake never fetches the pinned Dyninst. + - name: Check out this pull request into external/dyninst + uses: actions/checkout@v6 + with: + path: rocm-systems/projects/rocprofiler-systems/external/dyninst + + # rocprof-sys-instrument discards any module whose path matches the regex + # "dyninst" (is_module_constrained in module_function.cpp), and the module + # name is the full source path. GITHUB_WORKSPACE is /__w/dyninst/dyninst + # because this repository is named dyninst, so a build left there produces + # test binaries with zero functions instrumented: the rewrite succeeds, the + # tests run, and every trace assertion fails for a reason unrelated to + # Dyninst. No command-line flag overrides it, as the module check precedes + # the --min-instructions and --include filters. + - name: Move the checkout off the "dyninst" path + working-directory: ${{ github.workspace }} + run: | + rm -rf /rocm-systems + mv rocm-systems /rocm-systems + # -P because the compiler records the resolved path, not the logical one. + build_root=$(cd /rocm-systems/projects/rocprofiler-systems && pwd -P) + echo "building in ${build_root}" + if echo "${build_root}" | grep -qi dyninst; then + echo "Path still matches 'dyninst'; every test binary would be built uninstrumented." + exit 1 + fi + + - name: Record the versions under test + run: | + git config --global --add safe.directory '*' + test -f external/dyninst/CMakeLists.txt + dyninst_sha=$(git -C external/dyninst rev-parse HEAD) + echo "DYNINST_SHA=${dyninst_sha}" >> "${GITHUB_ENV}" + { + echo "| component | commit |" + echo "| --- | --- |" + echo "| rocm-systems ${ROCM_SYSTEMS_BRANCH} | \`${{ needs.matrix.outputs.sha }}\` |" + echo "| dyninst (this PR) | \`${dyninst_sha}\` |" + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Configure ROCm environment (Ubuntu / Debian) + if: ${{ !matrix.is_rhel }} + run: | + echo "/opt/rocm/bin" >> "${GITHUB_PATH}" + echo "ROCM_PATH=/opt/rocm" >> "${GITHUB_ENV}" + echo "LD_LIBRARY_PATH=/opt/rocm/lib:${LD_LIBRARY_PATH}" >> "${GITHUB_ENV}" + + - name: Configure ROCm environment (RHEL) + if: ${{ matrix.is_rhel }} + run: | + echo "CC=${{ matrix.cc }}" >> "${GITHUB_ENV}" + echo "CXX=${{ matrix.compiler }}" >> "${GITHUB_ENV}" + echo "/opt/rocm/bin" >> "${GITHUB_PATH}" + echo "/opt/rocm/llvm/bin" >> "${GITHUB_PATH}" + echo "ROCM_PATH=/opt/rocm" >> "${GITHUB_ENV}" + echo "LD_LIBRARY_PATH=/opt/rocm/lib:${LD_LIBRARY_PATH}" >> "${GITHUB_ENV}" + + # Upstream uses nick-fields/retry here. A loop keeps this repository free + # of third-party actions; the retries matter because the package mirrors + # are the flakiest step in the job. + - name: Install distro packages + env: + APT_COMPILER: ${{ matrix.apt_install_compiler && matrix.compiler || '' }} + APT_SYSTEM_DEPS: ${{ matrix.kind == 'system_deps' && matrix.system_deps_apt || '' }} + run: | + packages="${APT_COMPILER} ${APT_SYSTEM_DEPS}" + if [ "${{ matrix.is_rhel }}" != "true" ] && [ -z "${packages// /}" ]; then + echo "No extra packages needed for this entry" + exit 0 + fi + retry() { + for attempt in 1 2 3 4 5; do + if "$@"; then return 0; fi + echo "attempt ${attempt} failed, retrying in 30s" + sleep 30 + done + return 1 + } + if [ "${{ matrix.is_rhel }}" = "true" ]; then + # RHEL 8 ships glibc 2.28, but the trace_processor_shell that the + # perfetto python package downloads on demand needs 2.29, so every + # perfetto assertion fails before it reads the trace. The tests read + # ROCPROFSYS_TRACE_PROC_SHELL for exactly this case (see + # tests/pytest/rocprofsys/validators.py); v47.0 is the build + # upstream pins for it. + if [ "${{ matrix.os_major }}" = "8" ]; then + mkdir -p /opt/trace_processor/bin + retry curl -fsSL -o /opt/trace_processor/bin/trace_processor_shell \ + https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v47.0/linux-amd64/trace_processor_shell + chmod +x /opt/trace_processor/bin/trace_processor_shell + echo "ROCPROFSYS_TRACE_PROC_SHELL=/opt/trace_processor/bin/trace_processor_shell" >> "${GITHUB_ENV}" + fi + # mpich lives in the crb repository on RHEL 10. + if [ "${{ matrix.os_major }}" = "10" ]; then + retry dnf install -y --enablerepo=crb mpich mpich-devel + echo "/usr/lib64/mpich/bin" >> "${GITHUB_PATH}" + fi + elif [ -n "${packages// /}" ]; then + retry apt-get update + # shellcheck disable=SC2086 + retry apt-get install -y ${packages} + apt-get autoclean + fi + + - name: Restore ccache + uses: actions/cache@v6 + with: + path: ${{ github.workspace }}/.ccache + key: rps-ccache-${{ matrix.ccache_key_distro }}-${{ matrix.kind }}-${{ github.sha }} + restore-keys: | + rps-ccache-${{ matrix.ccache_key_distro }}-${{ matrix.kind }}- + + - name: Configure ccache + run: | + mkdir -p "${CCACHE_DIR}" + ccache --max-size=2G + ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime,pch_defines + ccache -z + + - name: Install Python test dependencies + timeout-minutes: 10 + run: | + for env_dir in /opt/conda/envs/${{ matrix.python_envs_glob }}/; do + if [ -d "${env_dir}" ] && [ -x "${env_dir}bin/python3" ]; then + echo "Installing requirements into ${env_dir}" + "${env_dir}bin/python3" -m pip install -r requirements.txt + fi + done + + # Mirrors the two upstream CMAKE_COMMON_FLAGS blocks. The system-deps + # entries carry no DISABLE_EXAMPLES of their own, so it belongs here. + - name: Select dependency flags + run: | + if [ "${{ matrix.kind }}" = "system_deps" ]; then + flags="-DROCPROFSYS_BUILD_TBB=OFF -DROCPROFSYS_BUILD_ELFUTILS=OFF -DROCPROFSYS_BUILD_LIBIBERTY=OFF" + flags="${flags} -DROCPROFSYS_BUILD_HIDDEN_VISIBILITY=ON -DROCPROFSYS_STRIP_LIBRARIES=OFF" + flags="${flags} -DROCPROFSYS_DISABLE_EXAMPLES=transpose;rccl;openmp-target;openmp-vv;videodecode;jpegdecode;network" + else + flags="-DROCPROFSYS_BUILD_TBB=ON -DROCPROFSYS_BUILD_ELFUTILS=ON -DROCPROFSYS_BUILD_LIBIBERTY=ON" + fi + echo "DEP_FLAGS=${flags}" >> "${GITHUB_ENV}" + + # The trailing -L narrows the suite to the tests that actually exercise + # Dyninst. binary_rewrite and runtime_instrument reach ctest as labels + # because tests/pytest/conftest.py emits the parametrised `mode` of each + # test as a CTest label. `instrument` is the marker on + # TestRocprofilerSystemsInstrument in tests/pytest/test_binaries.py, which + # drives the rocprof-sys-instrument CLI itself rather than a mutatee + # (--simulate, --exe-only, --max-library-functions and the like). Both + # that marker and the file-wide rocprof_binary appear nowhere else in the + # suite, so naming it adds those cases and nothing besides. -LE matches + # the upstream exclusion. + - name: Generate CI scripts + timeout-minutes: 5 + run: | + echo "CMake: $(cmake --version | head -n 1)" + echo "Compiler: $(${{ matrix.compiler }} --version | head -n 1)" + # DEP_FLAGS holds several cmake -D arguments and must word-split. + # shellcheck disable=SC2086 + python3 ./scripts/run-ci.py --stage generate \ + --name "${CDASH_NAME}" \ + --site GitHub-dyninst \ + --build-jobs "$(nproc)" \ + -B build \ + -- \ + -DCMAKE_C_COMPILER=${{ matrix.cc }} \ + -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} \ + -DCMAKE_BUILD_TYPE=Release \ + -DROCPROFSYS_BUILD_TESTING=ON \ + -DROCPROFSYS_BUILD_EXAMPLES=ON \ + -DROCPROFSYS_USE_PYTHON=ON \ + -DROCPROFSYS_BUILD_DYNINST=ON \ + -DROCPROFSYS_MAX_THREADS=64 \ + -DROCPROFSYS_PYTHON_PREFIX=/opt/conda/envs \ + ${DEP_FLAGS} \ + ${{ matrix.cmake_flags }} \ + -DROCPROFSYS_BUILD_NUMBER=${{ github.run_attempt }} \ + -- \ + -L "binary_rewrite|runtime_instrument|instrument" \ + -LE "network|gpu" + + - name: Configure + timeout-minutes: 20 + run: | + python3 ./scripts/run-ci.py --stage configure \ + --name "${CDASH_NAME}" \ + -B build + + # Without this, a failure of the early-return guard in + # rocprofiler_systems_checkout_git_submodule() would quietly build the + # pinned Dyninst and report a green result for the wrong code. + - name: Verify the pull request is what got configured + run: | + now=$(git -C external/dyninst rev-parse HEAD) + if [ "${now}" != "${DYNINST_SHA}" ]; then + echo "external/dyninst changed during configure: ${DYNINST_SHA} -> ${now}" + echo "CMake replaced the injected checkout, so this job is not testing the pull request." + exit 1 + fi + echo "ok external/dyninst is still ${now}" + + - name: Build + timeout-minutes: 90 + run: | + python3 ./scripts/run-ci.py --stage build \ + --name "${CDASH_NAME}" \ + -B build + + - name: Test + timeout-minutes: 60 + run: | + python3 ./scripts/run-ci.py --stage test \ + --name "${CDASH_NAME}" \ + -B build + + # trace_processor_shell is a server process the validation script talks to + # over HTTP, and upstream carries this same step because on RHEL it has + # been seen to outlive the run and hold the job open. + - name: Kill Perfetto + if: ${{ matrix.is_rhel && (success() || failure()) }} + continue-on-error: true + run: | + procs=$(pgrep trace_processor_shell || true) + if [ -n "${procs}" ]; then + # shellcheck disable=SC2086 + kill -9 ${procs} + fi + + - name: CDash link + if: always() + run: | + python3 ./scripts/run-ci.py --stage cdash-link \ + --name "${CDASH_NAME}" + + - name: ccache stats + if: always() + run: ccache -s + + - name: Upload JUnit test results + if: always() + uses: actions/upload-artifact@v7 + with: + name: junit-${{ matrix.kind }}-${{ strategy.job-index }} + path: /rocm-systems/projects/rocprofiler-systems/build/test-results.xml + if-no-files-found: ignore + + - name: Upload ctest logs + if: failure() + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + name: ctest-${{ matrix.kind }}-${{ strategy.job-index }}-log + path: /rocm-systems/projects/rocprofiler-systems/build/*.log + if-no-files-found: ignore + + - name: Upload test output + if: failure() + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + name: data-${{ matrix.kind }}-${{ strategy.job-index }}-files + path: | + /rocm-systems/projects/rocprofiler-systems/build/rocprofsys-tests-config/*.cfg + /rocm-systems/projects/rocprofiler-systems/build/rocprofsys-tests-output/**/*.txt + /rocm-systems/projects/rocprofiler-systems/build/rocprofsys-tests-output/**/*-instr*.json + if-no-files-found: ignore diff --git a/.github/workflows/spack-build.yaml b/.github/workflows/spack-build.yaml deleted file mode 100644 index 9a4b143926..0000000000 --- a/.github/workflows/spack-build.yaml +++ /dev/null @@ -1,19 +0,0 @@ -name: Spack Build - -on: - schedule: - - cron: '0 3 * * 0' # Every Sunday at 3AM - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: spack - run: | - sudo apt update -qq - sudo apt install -y -qq --no-install-recommends build-essential gcc g++ m4 cmake autoconf python3 git - git clone --depth=1 --branch=develop https://github.com/spack/spack - spack/bin/spack compiler find - spack/bin/spack external find --not-buildable cmake - spack/bin/spack install dyninst@master diff --git a/.github/workflows/system-libs.yaml b/.github/workflows/system-libs.yaml deleted file mode 100644 index 55cc360bef..0000000000 --- a/.github/workflows/system-libs.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# Do a simple parse of all system libraries - -name: Parse sys libs - -on: - schedule: - - cron: '0 1 * * 1' # 1AM on Monday - workflow_dispatch: - -jobs: - parse: - permissions: - packages: read - strategy: - fail-fast: false - matrix: - os: ['ubuntu-20.04', 'ubuntu-22.04', 'ubuntu-23.04', 'ubuntu-23.10', 'ubuntu-24.04', 'fedora-37', 'fedora-38', 'fedora-39'] - runs-on: ubuntu-latest - container: - image: ghcr.io/dyninst/amd64/${{ matrix.os }}:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.github_token }} - name: ${{ matrix.os }} - steps: - - name: Build parser - run: | - git clone --depth=1 https://github.com/dyninst/external-tests - cd external-tests - mkdir build - cd build - cmake .. -DDyninst_DIR=/dyninst/install/lib/cmake/Dyninst - cmake --build . - cp parseAPI/simpleParser / - - - name: Run parser - run: | - cd / - export LD_LIBRARY_PATH=/dyninst/install/lib:$LD_LIBRARY_PATH - for dir in /usr/lib /usr/lib64; do \ - for file in $(find $dir -type f -name "*.so.*"); do \ - echo $file; \ - ./simpleParser $file; \ - done \ - done diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml deleted file mode 100644 index dbc7a227f6..0000000000 --- a/.github/workflows/unit-tests.yaml +++ /dev/null @@ -1,53 +0,0 @@ -name: Unit tests - -on: - pull_request: - branches: - - master - workflow_dispatch: - -jobs: - unit-tests: - permissions: - packages: read - strategy: - fail-fast: false - matrix: - os: ['ubuntu-20.04', 'ubuntu-22.04', 'ubuntu-23.04', 'ubuntu-23.10', 'ubuntu-24.04', 'fedora-37', 'fedora-38', 'fedora-39'] - runs-on: ubuntu-latest - container: - image: ghcr.io/dyninst/amd64/${{ matrix.os }}-base:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.github_token }} - name: unit tests - steps: - - name: Checkout Dyninst - uses: actions/checkout@v3 - with: - path: dyninst/src - - - name: Build Dyninst - run: | - ln -s $PWD/dyninst /dyninst - export DYNINST_C_FLAGS="-Werror" DYNINST_CXX_FLAGS="-Werror" - export DYNINST_C_COMPILER="gcc" DYNINST_CXX_COMPILER="g++" - export EXTRA_CMAKE_FLAGS="-DDYNINST_EXPORT_ALL=1" - bash /dyninst/src/docker/build.sh /dyninst/src 2 - - - name: Checkout Unit Tests - uses: actions/checkout@v3 - with: - repository: dyninst/unit-tests - path: unit-tests - - - name: Build unit tests - run: | - cd unit-tests; mkdir build; cd build - cmake .. -DDyninst_DIR=/dyninst/install/lib/cmake/Dyninst -DDYNINST_SOURCE_TREE=/dyninst/src - cmake --build . --parallel 2 - - - name: Run unit tests - run: | - cd unit-tests/build - ctest . diff --git a/.gitignore b/.gitignore index 465fd49466..7600603c8a 100644 --- a/.gitignore +++ b/.gitignore @@ -121,6 +121,8 @@ doxyfiles/* .vscode/* build*/ cmake-build-*/ +# Staging prefix produced by scripts/build-tpls.sh +.tpls/ .project .cproject .settings diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..3367e5e341 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,51 @@ +# Formats and lints the files CI checks, so the lint workflow never asks for a +# manual fix. Enable it once per clone with: +# +# pip install pre-commit && pre-commit install +# +# CI runs these same hooks, so the pins below are the single definition of +# "correct" for this repository. +# +# Every hook is declared locally with a pinned pip package rather than pulled +# from an upstream hook repository. cheshirekow/cmake_format publishes no +# .pre-commit-hooks.yaml at all, and doing the same thing for the rest keeps one +# mechanism instead of three. + +repos: + - repo: local + hooks: + # The [YAML] extra is required: cmakelang imports PyYAML lazily and does + # not depend on it, so reading .cmake-format.yaml fails with + # ModuleNotFoundError without it. + - id: cmake-format + name: cmake-format + entry: cmake-format --in-place + language: python + additional_dependencies: ['cmake-format[YAML]==0.6.13'] + files: (^|/)CMakeLists\.txt$|\.cmake$ + + # Scoped to scripts/. The inherited docker/ and instructionAPI/ scripts + # have dozens of pre-existing findings, and rewriting them would churn + # files that still merge from upstream Dyninst for no CI benefit. + # + # --external-sources lets it follow `source`d files such as + # scripts/tpl-versions.env instead of reporting SC1091. + - id: shellcheck + name: shellcheck + entry: shellcheck --external-sources + language: python + additional_dependencies: ['shellcheck-py==0.11.0.1'] + files: ^scripts/ + types: [shell] + + # actionlint shells out to shellcheck to check inline `run:` blocks, which + # is where most of this repository's untested shell lives. It only does so + # when shellcheck is on PATH, hence the second dependency. + - id: actionlint + name: actionlint + entry: actionlint + language: python + additional_dependencies: + - 'actionlint-py==1.7.12.24' + - 'shellcheck-py==0.11.0.1' + files: ^\.github/workflows/.*\.ya?ml$ diff --git a/CMakeLists.txt b/CMakeLists.txt index 5174c479f0..ac632af7da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,17 @@ include_directories(SYSTEM "${PROJECT_SOURCE_DIR}") include(DyninstLibrarySettings) include(DyninstOptions) +# Installed public headers include (concurrent.h and most of the +# BPatch_* headers do), so the directory has to ship alongside them. Without +# this, the include_directories above makes the build work while anything +# compiling against the install tree fails to find the header. +install( + DIRECTORY "${PROJECT_SOURCE_DIR}/dyncompat" + DESTINATION "${DYNINST_INSTALL_INCLUDEDIR}" + FILES_MATCHING + PATTERN "*.h" + PATTERN "*.hpp") + # Set up Dyninst internals include(DyninstPlatform) include(DyninstCapArchDef) diff --git a/cmake/DyninstWarnings.cmake b/cmake/DyninstWarnings.cmake index 6e450f642f..51f0e31005 100644 --- a/cmake/DyninstWarnings.cmake +++ b/cmake/DyninstWarnings.cmake @@ -156,6 +156,9 @@ if(HAS_CPP_FLAG_Wframe_larger_than AND NOT DYNINST_DISABLE_DIAGNOSTIC_SUPPRESSIO set(debugMaxFrameSizeOverridePowerOpcodeTable 358400) if(${CMAKE_CXX_COMPILER_VERSION} MATCHES "^[7](\.|$)") set(nonDebugMaxFrameSizeOverridePowerOpcodeTable 38912) + elseif(${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER_EQUAL 14) + # gcc 14 emits a 74560-byte frame for the buildTables() lambda at -O2. + set(nonDebugMaxFrameSizeOverridePowerOpcodeTable 76800) endif() # most gcc's are under the default using -Og, but rhel's requires 30000 set(debugMaxFrameSizeOverrideFinalizeOperands 30000) diff --git a/common/h/Annotatable.h b/common/h/Annotatable.h index b5bc5cea14..ac1a149bf2 100644 --- a/common/h/Annotatable.h +++ b/common/h/Annotatable.h @@ -176,7 +176,7 @@ class COMMON_EXPORT AnnotatableDense if (annotations->data == NULL) { - annotations->data = (anno_list_t *) calloc(sizeof(anno_list_t), (size)); + annotations->data = (anno_list_t *) calloc(size, sizeof(anno_list_t)); annotations->max = size; for (unsigned i=0; idata[i] = NULL; @@ -228,7 +228,7 @@ class COMMON_EXPORT AnnotatableDense annotations = (aInfo *) malloc(sizeof(aInfo)); unsigned size = rhs.annotations->max; annotations->max = size; - annotations->data = (anno_list_t *)calloc(sizeof(anno_list_t), (size)); + annotations->data = (anno_list_t *)calloc(size, sizeof(anno_list_t)); memcpy(annotations->data, rhs.annotations->data, size * sizeof(anno_list_t)); } else { annotations = NULL; diff --git a/scripts/build-tpls.sh b/scripts/build-tpls.sh new file mode 100755 index 0000000000..20e621f522 --- /dev/null +++ b/scripts/build-tpls.sh @@ -0,0 +1,218 @@ +#!/usr/bin/env bash +# +# Build Dyninst's third-party libraries (oneTBB, elfutils, libiberty) from source. +# +# Dyninst's CMake requires all three to be present and has no download fallback: +# cmake/tpls/Dyninst{TBB,ElfUtils,LibIberty}.cmake each call find_package(REQUIRED). +# Distro packages lag well behind the versions Dyninst is validated against as part +# of rocprofiler-systems, so CI builds them here instead. See scripts/tpl-versions.env. +# +# Usage: +# build-tpls.sh --prefix DIR [--jobs N] [--skip-prereqs] +# +# Produces one root per library, mirroring the rocprofiler-systems layout so that +# binutils' generic headers (dwarf2.h, demangle.h, ...) cannot shadow elfutils': +# +# $PREFIX/tbb pass to cmake as -DTBB_ROOT_DIR +# $PREFIX/elfutils pass to cmake as -DElfUtils_ROOT_DIR +# $PREFIX/binutils pass to cmake as -DLibIberty_ROOT_DIR +# +# The prefix is self-describing: a stamp file records the versions it was built +# from, so a restored CI cache built from different versions is rebuilt rather +# than silently reused. + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source-path=SCRIPTDIR source=tpl-versions.env +source "${script_dir}/tpl-versions.env" + +prefix="" +jobs="$(nproc 2>/dev/null || echo 2)" +skip_prereqs=0 + +usage() { + sed -n '3,25p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --prefix) prefix="$2"; shift 2 ;; + --jobs|-j) jobs="$2"; shift 2 ;; + --skip-prereqs) skip_prereqs=1; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "error: unknown argument '$1'" >&2; usage >&2; exit 2 ;; + esac +done + +if [[ -z "${prefix}" ]]; then + echo "error: --prefix is required" >&2 + exit 2 +fi + +mkdir -p "${prefix}" +prefix="$(cd "${prefix}" && pwd)" + +tbb_root="${prefix}/tbb" +elfutils_root="${prefix}/elfutils" +binutils_root="${prefix}/binutils" + +stamp="${prefix}/.tpl-versions" +want_stamp="onetbb=${ONETBB_VERSION} elfutils=${ELFUTILS_VERSION} binutils=${BINUTILS_VERSION}" + +if [[ -f "${stamp}" ]] && [[ "$(cat "${stamp}")" == "${want_stamp}" ]]; then + echo "Third-party libraries already present at ${prefix} (${want_stamp}); nothing to do." + exit 0 +fi + +echo "Building third-party libraries into ${prefix}" +echo " ${want_stamp}" +echo " jobs: ${jobs}" + +install_prereqs_apt() { + apt-get update -qq + apt-get install -y -qq --no-install-recommends \ + bzip2 ca-certificates curl git m4 make pkg-config \ + zlib1g-dev libzstd-dev libbz2-dev liblzma-dev +} + +install_prereqs_dnf() { + # libzstd-devel ships in CodeReady Builder, which is disabled by default and + # named powertools on RHEL 8 but crb from RHEL 9 on. Rather than detect the + # name by parsing repolist -- whose output differs between dnf4 and dnf5 -- + # try the plain install first, so images that already enable it, or that + # carry the package in a base repository, are unaffected. + local pkgs=( + bzip2 ca-certificates curl git m4 make pkgconfig + zlib-devel libzstd-devel bzip2-devel xz-devel + ) + + if dnf install -y "${pkgs[@]}"; then + return 0 + fi + + local repo + for repo in crb powertools; do + echo "retrying prerequisite install with --enablerepo=${repo}" + if dnf install -y "--enablerepo=${repo}" "${pkgs[@]}"; then + return 0 + fi + done + + return 1 +} + +install_prereqs() { + if command -v apt-get >/dev/null 2>&1; then + install_prereqs_apt + elif command -v dnf >/dev/null 2>&1; then + install_prereqs_dnf + else + echo "error: no supported package manager (apt-get or dnf) found." >&2 + echo " Re-run with --skip-prereqs after installing the equivalents of:" >&2 + echo " bzip2 curl git m4 make pkg-config zlib libzstd libbz2 liblzma (all -dev)" >&2 + exit 1 + fi +} + +build_tbb() { + echo "::group::Build oneTBB ${ONETBB_VERSION}" + local src="${workdir}/oneTBB" + git clone --depth 1 --branch "v${ONETBB_VERSION}" \ + https://github.com/uxlfoundation/oneTBB.git "${src}" + + # TBB_TEST=OFF skips the (slow) test tree; TBB_STRICT=OFF keeps oneTBB's own + # -Werror from failing the build on whichever compiler the image ships. + cmake -S "${src}" -B "${workdir}/tbb-build" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${tbb_root}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DTBB_TEST=OFF \ + -DTBB_STRICT=OFF \ + -DTBB_DISABLE_HWLOC_AUTOMATIC_SEARCH=ON + cmake --build "${workdir}/tbb-build" --parallel "${jobs}" \ + --target tbb tbbmalloc tbbmalloc_proxy + cmake --install "${workdir}/tbb-build" + echo "::endgroup::" +} + +build_elfutils() { + echo "::group::Build elfutils ${ELFUTILS_VERSION}" + local tarball="elfutils-${ELFUTILS_VERSION}.tar.bz2" + local src="${workdir}/elfutils-${ELFUTILS_VERSION}" + + curl -fsSL --retry 3 --retry-delay 5 -o "${workdir}/${tarball}" \ + "https://sourceware.org/elfutils/ftp/${ELFUTILS_VERSION}/${tarball}" \ + || curl -fsSL --retry 3 --retry-delay 5 -o "${workdir}/${tarball}" \ + "https://mirrors.kernel.org/sourceware/elfutils/${ELFUTILS_VERSION}/${tarball}" + tar -xf "${workdir}/${tarball}" -C "${workdir}" + + # Flags mirror rocprofiler-systems' DyninstElfUtils.cmake. -fPIC because + # Dyninst links these into shared libraries. debuginfod is disabled to match + # Dyninst's ENABLE_DEBUGINFOD default of OFF; enabling one without the other + # produces a find_package component mismatch. + ( + cd "${src}" + CFLAGS="-fPIC -O3 -Wno-error=maybe-uninitialized" \ + CXXFLAGS="-fPIC -O3 -Wno-error=maybe-uninitialized" \ + LDFLAGS="-Wl,-rpath,${elfutils_root}/lib -pthread" \ + ./configure \ + --prefix="${elfutils_root}" \ + --libdir="${elfutils_root}/lib" \ + --enable-install-elfh \ + --enable-thread-safety \ + --disable-libdebuginfod \ + --disable-debuginfod \ + --disable-nls + make install "-j${jobs}" + ) + echo "::endgroup::" +} + +build_libiberty() { + echo "::group::Build libiberty from binutils ${BINUTILS_VERSION}" + local tarball="binutils-${BINUTILS_VERSION}.tar.gz" + local src="${workdir}/binutils-${BINUTILS_VERSION}" + + curl -fsSL --retry 3 --retry-delay 5 -o "${workdir}/${tarball}" \ + "https://ftpmirror.gnu.org/gnu/binutils/${tarball}" \ + || curl -fsSL --retry 3 --retry-delay 5 -o "${workdir}/${tarball}" \ + "https://mirrors.kernel.org/sourceware/binutils/releases/${tarball}" + tar -xf "${workdir}/${tarball}" -C "${workdir}" + + mkdir -p "${binutils_root}/lib" "${binutils_root}/include" + + # Build only the libiberty subtree: a full binutils build additionally needs + # bison/flex/texinfo. MAKEINFO=true no-ops the doc rules that would otherwise + # require Texinfo. + ( + cd "${src}" + CFLAGS="-fPIC -O3 -Wno-error" \ + CXXFLAGS="-fPIC -O3 -Wno-error" \ + MAKEINFO=true \ + ./configure --prefix="${binutils_root}" + make MAKEINFO=true "-j${jobs}" all-libiberty + ) + + install -C "${src}/libiberty/libiberty.a" "${binutils_root}/lib/" + install -C -m 644 "${src}"/include/*.h "${binutils_root}/include/" + echo "::endgroup::" +} + +workdir="$(mktemp -d)" +trap 'rm -rf "${workdir}"' EXIT + +if [[ "${skip_prereqs}" -eq 0 ]]; then + install_prereqs +fi + +build_tbb +build_elfutils +build_libiberty + +echo "${want_stamp}" > "${stamp}" + +echo "Third-party libraries installed:" +echo " TBB_ROOT_DIR = ${tbb_root}" +echo " ElfUtils_ROOT_DIR = ${elfutils_root}" +echo " LibIberty_ROOT_DIR = ${binutils_root}" diff --git a/scripts/tpl-versions.env b/scripts/tpl-versions.env new file mode 100644 index 0000000000..1a1d062e93 --- /dev/null +++ b/scripts/tpl-versions.env @@ -0,0 +1,17 @@ +# Versions of Dyninst's third-party libraries built from source by build-tpls.sh. +# +# These deliberately mirror what rocprofiler-systems builds, so that CI validates +# Dyninst against the dependency versions it is actually shipped with rather than +# whatever the distro happens to package. When bumping any of these, check the +# corresponding pin downstream: +# +# oneTBB .gitmodules -> projects/rocprofiler-systems/external/onetbb +# elfutils projects/rocprofiler-systems/cmake/DyninstElfUtils.cmake +# binutils projects/rocprofiler-systems/cmake/DyninstLibIberty.cmake +# +# This file is read by build-tpls.sh and is the cache key for the built prefix, +# so any edit here invalidates the CI cache and forces a rebuild. + +ONETBB_VERSION=2022.3.0 +ELFUTILS_VERSION=0.195 +BINUTILS_VERSION=2.46.0 diff --git a/symtabAPI/src/indexed_symbols.hpp b/symtabAPI/src/indexed_symbols.hpp index 80b0c85540..f87e13d306 100644 --- a/symtabAPI/src/indexed_symbols.hpp +++ b/symtabAPI/src/indexed_symbols.hpp @@ -71,28 +71,32 @@ struct indexed_symbols { if (!by_offset.find(oa, s->getOffset())) { assert(!"by_offset.find(oa, s->getOffset())"); } - std::remove(oa->second.begin(), oa->second.end(), s); + auto it = std::remove(oa->second.begin(), oa->second.end(), s); + oa->second.erase(it, oa->second.end()); } { by_name_t::accessor ma; if (!by_mangled.find(ma, s->getMangledName())) { assert(!"by_mangled.find(ma, s->getMangledName())"); } - std::remove(ma->second.begin(), ma->second.end(), s); + auto it = std::remove(ma->second.begin(), ma->second.end(), s); + ma->second.erase(it, ma->second.end()); } { by_name_t::accessor pa; if (!by_pretty.find(pa, s->getPrettyName())) { assert(!"by_pretty.find(pa, s->getPrettyName())"); } - std::remove(pa->second.begin(), pa->second.end(), s); + auto it = std::remove(pa->second.begin(), pa->second.end(), s); + pa->second.erase(it, pa->second.end()); } { by_name_t::accessor ta; if (!by_typed.find(ta, s->getTypedName())) { assert(!"by_typed.find(ta, s->getTypedName())"); } - std::remove(ta->second.begin(), ta->second.end(), s); + auto it = std::remove(ta->second.begin(), ta->second.end(), s); + ta->second.erase(it, ta->second.end()); } } }