Build & Test Linux x86_64 workflow (chdb-io/chdb)

name: Build & Test Linux x86_64
 
on:
  workflow_dispatch:
    inputs:
      TAG_NAME:
        description: 'Release Version Tag'
        required: true
  release:
    types: [created]
  push:
    branches:
      - main
    paths-ignore:
      - '**/*.md'
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review]
    branches:
      - main
    paths-ignore:
      - '**/*.md'
 
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
 
jobs:
  build_and_test:
    timeout-minutes: 30
    name: Build & Test (Linux x86_64)
    runs-on: [self-hosted, linux, x64, ubuntu-latest]
    if: ${{ !github.event.pull_request.draft }}
    steps:
      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \
            libbz2-dev libreadline-dev wget curl \
            libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \
            libffi-dev liblzma-dev libsqlite3-dev
      - name: Install GitHub CLI
        run: |
          wget https://github.com/cli/cli/releases/download/v2.82.1/gh_2.82.1_linux_amd64.tar.gz -O gh.tar.gz
          tar -xf gh.tar.gz
          sudo cp gh_*/bin/gh /usr/local/bin/
          sudo chmod +x /usr/local/bin/gh
          if ! gh --version; then
            echo "ERROR: GitHub CLI installation failed!"
            exit 1
          fi
          echo "GitHub CLI installed successfully"
      - name: Setup pyenv
        run: |
          rm -rf $HOME/.pyenv
          curl https://pyenv.run | bash
          export PATH="$HOME/.pyenv/bin:$PATH"
          eval "$(pyenv init -)"
          INSTALLED=""
          set +e
          for v in 3.9 3.10 3.11 3.12 3.13 3.14; do
            echo "=== Installing Python $v ==="
            if pyenv install "$v:latest"; then
              INSTALLED="$INSTALLED $v"
            else
              echo "Normal install failed, retrying without ensurepip..."
              installed_ver=$(pyenv latest "$v" 2>/dev/null)
              if [ -n "$installed_ver" ]; then pyenv uninstall -f "$installed_ver" 2>/dev/null; fi
              PYTHON_CONFIGURE_OPTS="--without-ensurepip" pyenv install "$v:latest" || true
              if pyenv versions --bare | grep -q "^$v\."; then
                INSTALLED="$INSTALLED $v"
                echo "Python $v binary installed (pip bootstrap deferred)"
              else
                echo "WARNING: Python $v install failed completely, skipping"
              fi
            fi
          done
          set -e
          if [ -z "$INSTALLED" ]; then echo "No Python installed!"; exit 1; fi
          pyenv global $INSTALLED
 
          echo "Installed versions:"
          pyenv versions
      - name: Install dependencies for all Python versions
        run: |
          export PATH="$HOME/.pyenv/bin:$PATH"
          eval "$(pyenv init -)"
          for version in 3.9 3.10 3.11 3.12 3.13 3.14; do
            if ! pyenv versions --bare | grep -q "^$version"; then continue; fi
            echo "Installing dependencies for Python $version"
            pyenv shell $version
            if ! python -m pip --version 2>/dev/null; then
              echo "pip missing, bootstrapping..."
              curl -sS https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
              python /tmp/get-pip.py || python -m ensurepip --upgrade || true
            fi
            python -m pip install --upgrade pip
            python -m pip install setuptools tox pandas pyarrow twine wheel pytest pytest-cov
            pyenv shell --unset
          done
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: Update version for release
        if: startsWith(github.ref, 'refs/tags/v')
        run: |
          export PATH="$HOME/.pyenv/bin:$PATH"
          eval "$(pyenv init -)"
          pyenv shell 3.9
 
          python -m pip install bump-my-version
          TAG_NAME=${GITHUB_REF#refs/tags/v}
          bump-my-version replace --new-version $TAG_NAME
          echo "Version files updated to $TAG_NAME"
          pyenv shell --unset
      - name: Build wheel
        run: |
          export PATH="$HOME/.pyenv/bin:$PATH"
          eval "$(pyenv init -)"
          pyenv shell 3.9
          make wheel
      - name: Show files
        run: |
          ls -lh dist
        shell: bash
      - name: Test DataStore on all Python versions with pandas 2.x and 3.x
        run: |
          export PATH="$HOME/.pyenv/bin:$PATH"
          eval "$(pyenv init -)"
 
          for version in 3.9 3.10 3.11 3.12 3.13 3.14; do
            if ! pyenv versions --bare | grep -q "^$version"; then
              echo "Python $version not available, skipping"
              continue
            fi
            # pandas 3.x requires Python >= 3.11
            if [[ "$version" == "3.9" || "$version" == "3.10" ]]; then
              PANDAS_CONSTRAINTS=("pandas<3.0")
            else
              PANDAS_CONSTRAINTS=("pandas<3.0" "pandas>=3.0")
            fi
            for PANDAS_CONSTRAINT in "${PANDAS_CONSTRAINTS[@]}"; do
              echo "=============================================="
              echo "Testing DataStore on Python $version with $PANDAS_CONSTRAINT"
              echo "=============================================="
 
              pyenv shell $version
              python -m pip install dist/*.whl --force-reinstall
              python -m pip install pytest pytest-cov hypothesis ray "$PANDAS_CONSTRAINT"
              echo "Installed pandas version:"
              python -c "import pandas; print(pandas.__version__)"
 
              echo "Running DataStore tests on Python $version with $PANDAS_CONSTRAINT ..."
              cd datastore
              python -m pytest tests/ -v --tb=short -x
              cd ..
              echo "=============================================="
              echo "DataStore tests PASSED on Python $version with $PANDAS_CONSTRAINT"
              echo "=============================================="
              pyenv shell --unset
            done
          done
        continue-on-error: false
      - name: Upload wheels to release
        if: startsWith(github.ref, 'refs/tags/v')
        run: |
          gh release upload ${{ github.ref_name }} dist/*.whl --clobber
        env:
          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
      - uses: actions/upload-artifact@v4
        with:
          name: chdb-wheel
          path: ./dist/*.whl
          overwrite: true
      - name: Upload pypi
        if: startsWith(github.ref, 'refs/tags/v')
        run: |
          export PATH="$HOME/.pyenv/bin:$PATH"
          eval "$(pyenv init -)"
          pyenv shell 3.9
          python -m twine upload dist/*.whl
        env:
          TWINE_USERNAME: __token__
          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}