Python.yml

name: Python
on:
  workflow_dispatch:
  repository_dispatch:
  push:
    branches:
      - '**'
      - '!main'
      - '!feature'
    tags:
      - '**'
    paths-ignore:
      - '**.md'
      - 'examples/**'
      - 'test/**'
      - 'tools/**'
      - '!tools/pythonpkg/**'
      - '.github/patches/duckdb-wasm/**'
      - '.github/workflows/**'
      - '!.github/workflows/Python.yml'

  pull_request:
    types: [opened, reopened, ready_for_review]
    paths-ignore:
      - '**.md'
      - 'examples/**'
      - 'test/**'
      - 'tools/**'
      - '!tools/pythonpkg/**'
      - '.github/patches/duckdb-wasm/**'
      - '.github/workflows/**'
      - '!.github/workflows/Python.yml'


concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
  cancel-in-progress: true

env:
  GH_TOKEN: ${{ secrets.GH_TOKEN }}

jobs:
#  This is just a sanity check of Python 3.9 running with Arrow
   linux-python3-9:
    name: Python 3.9 Linux
    runs-on: ubuntu-20.04

    env:
      CIBW_BUILD: 'cp39-manylinux_x86_64'
      CIBW_TEST_COMMAND: 'python -m pytest {project}/tests --verbose'
      PYTEST_TIMEOUT: '600'

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - uses: actions/setup-python@v4
      with:
        python-version: '3.10'

    - name: Install
      shell: bash
      run: pip install 'cibuildwheel>=2.16.2' twine build

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@main
      with:
        key: ${{ github.job }}
        save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}

    - name: Build source dist
      shell: bash
      working-directory: tools/pythonpkg
      run: |
        pyproject-build . --sdist
        mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball
    - name: Build
      shell: bash
      working-directory: tools/pythonpkg
      run: |
        export DISTUTILS_C_COMPILER_LAUNCHER=ccache
        # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
        cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball

   manylinux-extensions-x64:
    name: Linux Extensions (linux_amd64_gcc4)
    runs-on: ubuntu-latest
    container: quay.io/pypa/manylinux2014_x86_64
    needs: linux-python3-9
    env:
      GEN: ninja

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - uses: ./.github/actions/manylinux_2014_setup
      with:
        aws-cli: 1
        ninja-build: 1
        ccache: 1
        nodejs: 1
        ssh: 1
        python_alias: 1
        openssl: 1

    - uses: ./.github/actions/build_extensions
      with:
        vcpkg_target_triplet: x64-linux
        post_install: rm build/release/src/libduckdb*
        deploy_as: linux_amd64_gcc4
        s3_id: ${{ secrets.S3_ID }}
        s3_key: ${{ secrets.S3_KEY }}
        signing_pk: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }}
        run_tests: ${{ startsWith(github.ref, 'refs/tags/v') && 0 || 1  }}
        run_autoload_tests: ${{ startsWith(github.ref, 'refs/tags/v') && 0 || 1  }}
        treat_warn_as_error: 0
        ninja: 1

    - uses: actions/upload-artifact@v3
      with:
        name: manylinux-extensions-x64
        path: |
          build/release/extension/*/*.duckdb_extension

   linux-python3:
    name: Python 3 Linux
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        arch: [x86_64, aarch64]
        python_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*, cp312-*]
        manylinux: [manylinux2014, manylinux_2_28]
        isRelease:
          - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
        exclude:
          # For now we don't distribute the 2_28 wheels on x64
          - arch: x86_64
            manylinux: manylinux_2_28
          # Speed things up a bit for non-releases
          - isRelease: false
            python_build: 'cp37-*'
          - isRelease: false
            python_build: 'cp38-*'
          - isRelease: false
            python_build: 'cp39-*'
          - isRelease: false
            python_build: 'cp310-*'
          - isRelease: false
            python_build: 'cp311-*'
          - isRelease: false
            arch: aarch64
    needs: manylinux-extensions-x64
    env:
      CIBW_BUILD: ${{ matrix.python_build}}
      CIBW_SKIP: '*-musllinux_aarch64'
      CIBW_ARCHS: ${{ matrix.arch == 'aarch64' && 'aarch64' || 'auto64' }}
      CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux }}
      CIBW_MANYLINUX_PYPY_AARCH64_IMAGE: ${{ matrix.manylinux }}
      CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux }}
      CIBW_MANYLINUX_PYPY_X86_64_IMAGE: ${{ matrix.manylinux }}
      PYTEST_TIMEOUT: '600'
      DUCKDB_BUILD_UNITY: 1

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - name: Set up QEMU
      uses: docker/setup-qemu-action@v2
      if: ${{ matrix.arch == 'aarch64' }}

    - uses: actions/setup-python@v4
      with:
        python-version: '3.10'

    - name: Install
      shell: bash
      run: |
        pip install 'cibuildwheel>=2.16.2' twine build
        python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true"

    - uses: actions/download-artifact@v3
      if: ${{ matrix.arch == 'x86_64' }}
      with:
        name: manylinux-extensions-x64
        path: tools/pythonpkg

    - name: List extensions to be tested
      shell: bash
      run: |
        find tools/pythonpkg -maxdepth 2 -type f -name "*.duckdb_extension"

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@main
      with:
        key: ${{ github.job }}-${{ matrix.arch }}-${{ matrix.python_build }}
        save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}

    - name: Build
      shell: bash
      run: |
        cd tools/pythonpkg
        pyproject-build . --sdist
        mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball
        ls duckdb_tarball
        export DISTUTILS_C_COMPILER_LAUNCHER=ccache
        # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
        if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then
          export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py  --verbose'
        fi
        cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball
        ls wheelhouse

    - name: Deploy
      env:
        TWINE_USERNAME: '__token__'
        TWINE_PASSWORD: ${{ secrets.TWINE_TOKEN }}
      shell: bash
      run: |
        python scripts/asset-upload-gha.py duckdb_python_src.tar.gz=tools/pythonpkg/dist/duckdb-*.tar.gz
        if [[ "$GITHUB_REF" =~ ^(refs/heads/main|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
          twine upload --non-interactive --disable-progress-bar --skip-existing tools/pythonpkg/wheelhouse/*.whl tools/pythonpkg/dist/duckdb-*.tar.gz
        fi

   osx-python3:
      if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main'
      name: Python 3 OSX
      runs-on: macos-latest
      strategy:
       matrix:
        python_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*, cp312-*]
      needs: linux-python3-9
      env:
        CIBW_BUILD: ${{ matrix.python_build}}
        CIBW_ARCHS: 'x86_64 universal2 arm64'
        CIBW_TEST_COMMAND: 'python -m pytest {project}/tests/fast  --verbose'
        TWINE_USERNAME: '__token__'
        DUCKDB_BUILD_UNITY: 1

      steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Install
        shell: bash
        run: pip install 'cibuildwheel>=2.16.2' twine build

      - name: Setup Ccache
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}-${{ matrix.python_build }}
          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}

      - name: Build
        shell: bash
        run: |
          cd tools/pythonpkg
          pyproject-build . --sdist
          mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball
          export DISTUTILS_C_COMPILER_LAUNCHER=ccache
          # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
          if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then
            export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py  --verbose'
          fi
          cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball

      - name: Deploy
        env:
          TWINE_USERNAME: '__token__'
          TWINE_PASSWORD: ${{ secrets.TWINE_TOKEN }}
        shell: bash
        run: |
          if [[ "$GITHUB_REF" =~ ^(refs/heads/main|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
            twine upload --non-interactive --disable-progress-bar --skip-existing tools/pythonpkg/wheelhouse/*.whl
          fi

   win-python3:
      name: Python 3 Windows
      runs-on: windows-2019
      strategy:
       matrix:
        python_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*, cp312-*]
        isRelease:
          - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
        exclude:
          - isRelease: false
            python_build: 'cp37-*'
          - isRelease: false
            python_build: 'cp38-*'
          - isRelease: false
            python_build: 'cp39-*'
          - isRelease: false
            python_build: 'cp311-*'
      needs: linux-python3-9

      env:
        CIBW_BUILD: ${{ matrix.python_build}}
        TWINE_USERNAME: '__token__'
        DUCKDB_BUILD_UNITY: 1

      steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Install
        shell: bash
        run: pip install 'cibuildwheel>=2.16.2' twine build

      - name: Setup Ccache
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}-${{ matrix.python_build }}
          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}

      - name: Build source dist
        shell: bash
        working-directory: tools/pythonpkg
        run: |
          pyproject-build . --sdist
          mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball

      - name: Build wheels
        shell: bash
        working-directory: tools/pythonpkg
        run: |
          export DISTUTILS_C_COMPILER_LAUNCHER=ccache
          # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
          if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then
            export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py  --verbose'
          fi
          cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball

      - name: Deploy
        env:
          TWINE_USERNAME: '__token__'
          TWINE_PASSWORD: ${{ secrets.TWINE_TOKEN }}
        shell: bash
        run: |
          if [[ "$GITHUB_REF" =~ ^(refs/heads/main|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
            twine upload --non-interactive --disable-progress-bar --skip-existing tools/pythonpkg/wheelhouse/*.whl
          fi

   linux-release-cleanup:
     if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main'
     name: PyPi Release Cleanup
     runs-on: ubuntu-20.04
     needs: linux-python3-9
     env:
       PYPI_CLEANUP_USERNAME: 'mytherin'
       PYPI_CLEANUP_PASSWORD: ${{secrets.PYPI_CLEANUP_PASSWORD}}
       PYPI_CLEANUP_OTP: ${{secrets.PYPI_CLEANUP_OTP}}

     steps:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0

     - uses: actions/setup-python@v4
       with:
         python-version: '3.10'

     - name: Install Packages
       shell: bash
       run: python3 -m pip install pyotp requests

     - name: Cleanup Releases
       shell: bash
       run: python3 scripts/pypi_cleanup.py