Merge branch 'main' into imaplib-idle

Author: Forest

.github/CODEOWNERS

@@ -5,11 +5,11 @@
 # https://git-scm.com/docs/gitignore#_pattern_format
 
 # GitHub
-.github/**                    @ezio-melotti @hugovk
+.github/**                    @ezio-melotti @hugovk @AA-Turner
 
 # pre-commit
 .pre-commit-config.yaml       @hugovk @AlexWaygood
-.ruff.toml                    @hugovk @AlexWaygood
+.ruff.toml                    @hugovk @AlexWaygood @AA-Turner
 
 # Build system
 configure*                    @erlend-aasland @corona10
@@ -56,6 +56,14 @@ Tools/c-analyzer/             @ericsnowcurrently
 # dbm
 **/*dbm*                      @corona10 @erlend-aasland @serhiy-storchaka
 
+# Doc/ tools
+Doc/conf.py                   @AA-Turner @hugovk
+Doc/Makefile                  @AA-Turner @hugovk
+Doc/make.bat                  @AA-Turner @hugovk
+Doc/requirements.txt          @AA-Turner @hugovk
+Doc/_static/**                @AA-Turner @hugovk
+Doc/tools/**                  @AA-Turner @hugovk
+
 # runtime state/lifecycle
 **/*pylifecycle*              @ericsnowcurrently
 **/*pystate*                  @ericsnowcurrently
@@ -96,13 +104,14 @@ Doc/library/site.rst          @FFY00
 Lib/test/test_except*.py      @iritkatriel
 Objects/exceptions.c          @iritkatriel
 
-# Hashing
-**/*hashlib*                  @gpshead @tiran
+# Hashing & cryptographic primitives
+**/*hashlib*                  @gpshead @tiran @picnixz
 **/*pyhash*                   @gpshead @tiran
-**/sha*                       @gpshead @tiran
-Modules/md5*                  @gpshead @tiran
-**/*blake*                    @gpshead @tiran
+**/sha*                       @gpshead @tiran @picnixz
+Modules/md5*                  @gpshead @tiran @picnixz
+**/*blake*                    @gpshead @tiran @picnixz
 Modules/_hacl/**              @gpshead
+**/*hmac*                     @gpshead @picnixz
 
 # logging
 **/*logging*                  @vsajip
@@ -292,6 +301,10 @@ Lib/configparser.py           @jaraco
 Lib/test/test_configparser.py @jaraco
 
 # Doc sections
-Doc/reference/                @willingc
+Doc/reference/                @willingc @AA-Turner
 
 **/*weakref*                  @kumaraditya303
+
+# Colorize
+Lib/_colorize.py              @hugovk
+Lib/test/test__colorize.py    @hugovk

.github/actionlint.yaml

@@ -1,10 +1,10 @@
 self-hosted-runner:
-  labels: ["ubuntu-24.04-aarch64", "windows-aarch64"]
+  labels: ["windows-aarch64"]
 
 config-variables: null
 
 paths:
   .github/workflows/**/*.yml:
      ignore:
      - 1st argument of function call is not assignable
-     - SC2(015|038|086|091|097|098|129|155)
+     - SC2(015|038|086|091|097|098|129|155)

.github/workflows/build.yml

@@ -18,6 +18,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable
   cancel-in-progress: true
 
+env:
+  FORCE_COLOR: 1
+
 jobs:
   check_source:
     name: Change detection
@@ -46,7 +49,7 @@ jobs:
     # reproducible: to get the same tools versions (autoconf, aclocal, ...)
     runs-on: ubuntu-24.04
     container:
-      image: ghcr.io/python/autoconf:2024.11.11.11786316759
+      image: ghcr.io/python/autoconf:2025.01.02.12581854023
     timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
@@ -58,11 +61,12 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 1
+          persist-credentials: false
       - name: Runner image version
         run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV"
       - name: Check Autoconf and aclocal versions
         run: |
-          grep "Generated by GNU Autoconf 2.71" configure
+          grep "Generated by GNU Autoconf 2.72" configure
           grep "aclocal 1.16.5" aclocal.m4
           grep -q "runstatedir" configure
           grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4
@@ -94,6 +98,8 @@ jobs:
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - uses: actions/setup-python@v5
         with:
           python-version: '3.x'
@@ -120,7 +126,7 @@ jobs:
       - name: Build CPython
         run: |
           make -j4 regen-all
-          make regen-stdlib-module-names regen-sbom
+          make regen-stdlib-module-names regen-sbom regen-unicodedata
       - name: Check for changes
         run: |
           git add -u
@@ -228,17 +234,33 @@ jobs:
     name: >-
       Ubuntu
       ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
+      ${{ fromJSON(matrix.bolt) && '(bolt)' || '' }}
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     strategy:
       matrix:
+        bolt:
+        - false
+        - true
         free-threading:
         - false
         - true
+        os:
+        - ubuntu-24.04
+        - ubuntu-24.04-arm
+        exclude:
+        # Do not test BOLT with free-threading, to conserve resources
+        - bolt: true
+          free-threading: true
+        # BOLT currently crashes during instrumentation on aarch64
+        - os: ubuntu-24.04-arm
+          bolt: true
     uses: ./.github/workflows/reusable-ubuntu.yml
     with:
       config_hash: ${{ needs.check_source.outputs.config_hash }}
+      bolt-optimizations: ${{ matrix.bolt }}
       free-threading: ${{ matrix.free-threading }}
+      os: ${{ matrix.os }}
 
   build_ubuntu_ssltests:
     name: 'Ubuntu SSL tests with OpenSSL'
@@ -259,6 +281,8 @@ jobs:
       LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib
     steps:
     - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
     - name: Runner image version
       run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV"
     - name: Restore config.cache
@@ -319,6 +343,8 @@ jobs:
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies
@@ -402,7 +428,7 @@ jobs:
         #
         # (GH-104097) test_sysconfig is skipped because it has tests that are
         # failing when executed from inside a virtual environment.
-        ${{ env.VENV_PYTHON }} -m test \
+        "${VENV_PYTHON}" -m test \
           -W \
           -o \
           -j4 \
@@ -437,6 +463,8 @@ jobs:
       ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
     steps:
     - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
     - name: Runner image version
       run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV"
     - name: Restore config.cache
@@ -484,26 +512,59 @@ jobs:
       run: xvfb-run make ci
 
   build_tsan:
-    name: 'Thread sanitizer'
+    name: >-
+      Thread sanitizer
+      ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
+    strategy:
+      matrix:
+        free-threading:
+        - false
+        - true
     uses: ./.github/workflows/reusable-tsan.yml
     with:
       config_hash: ${{ needs.check_source.outputs.config_hash }}
-      options: ./configure --config-cache --with-thread-sanitizer --with-pydebug
-      suppressions_path: Tools/tsan/supressions.txt
-      tsan_logs_artifact_name: tsan-logs-default
+      free-threading: ${{ matrix.free-threading }}
 
-  build_tsan_free_threading:
-    name: 'Thread sanitizer (free-threading)'
+  cross-build-linux:
+    name: Cross build Linux
+    runs-on: ubuntu-latest
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
-    uses: ./.github/workflows/reusable-tsan.yml
-    with:
-      config_hash: ${{ needs.check_source.outputs.config_hash }}
-      options: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug
-      suppressions_path: Tools/tsan/suppressions_free_threading.txt
-      tsan_logs_artifact_name: tsan-logs-free-threading
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Runner image version
+        run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV"
+      - name: Restore config.cache
+        uses: actions/cache@v4
+        with:
+          path: config.cache
+          key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
+      - name: Register gcc problem matcher
+        run: echo "::add-matcher::.github/problem-matchers/gcc.json"
+      - name: Set build dir
+        run:
+          # an absolute path outside of the working directoy
+          echo "BUILD_DIR=$(realpath ${{ github.workspace }}/../build)" >> "$GITHUB_ENV"
+      - name: Install Dependencies
+        run: sudo ./.github/workflows/posix-deps-apt.sh
+      - name: Configure host build
+        run: ./configure --prefix="$BUILD_DIR/host-python"
+      - name: Install host Python
+        run: make -j8 install
+      - name: Run test subset with host build
+        run: |
+          "$BUILD_DIR/host-python/bin/python3" -m test test_sysconfig test_site test_embed
+      - name: Configure cross build
+        run: ./configure --prefix="$BUILD_DIR/cross-python" --with-build-python="$BUILD_DIR/host-python/bin/python3"
+      - name: Install cross Python
+        run: make -j8 install
+      - name: Run test subset with host build
+        run: |
+          "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed
 
   # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
   cifuzz:
@@ -563,7 +624,6 @@ jobs:
     - test_hypothesis
     - build_asan
     - build_tsan
-    - build_tsan_free_threading
     - cifuzz
 
     runs-on: ubuntu-latest
@@ -597,7 +657,6 @@ jobs:
             build_windows,
             build_asan,
             build_tsan,
-            build_tsan_free_threading,
             '
             || ''
           }}

.github/workflows/documentation-links.yml

@@ -10,16 +10,16 @@ on:
     - 'Doc/**'
     - '.github/workflows/doc.yml'
 
-permissions:
-  pull-requests: write
-
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
 jobs:
   documentation-links:
     runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+
     steps:
       - uses: readthedocs/actions/preview@v1
         with:

.github/workflows/jit.yml

@@ -25,13 +25,18 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+env:
+  FORCE_COLOR: 1
+
 jobs:
   interpreter:
     name: Interpreter (Debug)
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 90
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: Build tier two interpreter
         run: |
           ./configure --enable-experimental-jit=interpreter --with-pydebug
@@ -78,12 +83,14 @@ jobs:
             runner: macos-14
           - target: x86_64-unknown-linux-gnu/gcc
             architecture: x86_64
-            runner: ubuntu-22.04
+            runner: ubuntu-24.04
           - target: aarch64-unknown-linux-gnu/gcc
             architecture: aarch64
-            runner: ubuntu-22.04
+            runner: ubuntu-24.04-arm
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
@@ -104,7 +111,7 @@ jobs:
 
         # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966.
         # This is a bug in the macOS runner image where the pre-installed Python is installed in the same
-        # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes 
+        # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes
         # the symlink to the pre-installed Python so that the Homebrew Python is used instead.
       - name: Native macOS
         if: runner.os == 'macOS'
@@ -118,43 +125,26 @@ jobs:
           ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       - name: Native Linux
-        if: runner.os == 'Linux' && matrix.architecture == 'x86_64'
+        if: runner.os == 'Linux'
         run: |
           sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
           export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
           ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }}
           make all --jobs 4
           ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
-      - name: Emulated Linux
-        if: runner.os == 'Linux' && matrix.architecture != 'x86_64'
-        # The --ignorefile on ./python -m test is used to exclude tests known to fail when running on an emulated Linux.
-        run: |
-          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
-          export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
-          ./configure --prefix="$(pwd)/../build"
-          make install --jobs 4
-          make clean --jobs 4
-          export HOST=${{ matrix.architecture }}-linux-gnu
-          sudo apt install --yes "gcc-$HOST" qemu-user
-          export QEMU_LD_PREFIX="/usr/$HOST"
-          CC="$HOST-gcc" \
-            CPP="$HOST-gcc --preprocess" \
-            HOSTRUNNER=qemu-${{ matrix.architecture }} \
-            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
-          make all --jobs 4
-          ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3
-
   jit-with-disabled-gil:
     name: Free-Threaded (Debug)
     needs: interpreter
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         llvm:
           - 19
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - uses: actions/setup-python@v5
         with:
           python-version: '3.11'