Studio Tests workflow (datachain-ai/datachain)

name: Studio Tests
 
on:
  push:
    branches: [main]
  pull_request:
  workflow_dispatch:
 
env:
  FORCE_COLOR: "1"
  BRANCH: ${{ github.head_ref || github.ref_name }}
 
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true
 
jobs:
  studio:
    timeout-minutes: 30
    if: '!github.event.pull_request.head.repo.fork'
    runs-on: latchkey-small
    strategy:
      matrix:
        pyv: ['3.13']
        group: [1, 2, 3, 4, 5, 6]
    services:
      postgres:
        image: postgres:16.3
        ports:
          - 5432:5432
        env:
          POSTGRES_USER: test
          POSTGRES_DB: database
          POSTGRES_HOST_AUTH_METHOD: trust
        options: >-
          --add-host=host.docker.internal:host-gateway
      clickhouse:
        image: clickhouse/clickhouse-server:25.8
        ports:
          - 8123:8123
          - 9010:9000
        env:
          CLICKHOUSE_DB: studio_local_db
          CLICKHOUSE_USER: studio_local
          CLICKHOUSE_PASSWORD: ch123456789!
          CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
        options: >-
          --add-host=host.docker.internal:host-gateway
      redis:
        image: redis:7.2.5
        ports:
          - 6379:6379
        options: >-
          --add-host=host.docker.internal:host-gateway
    steps:
      - name: Studio branch name
        env:
          BRANCH: ${{ env.BRANCH }}
          STUDIO_READ_ACCESS_TOKEN: ${{ secrets.ITERATIVE_STUDIO_READ_ACCESS_TOKEN }}
        run: |
          echo "DataChain branch: $BRANCH"
          if git ls-remote --heads https://"$STUDIO_READ_ACCESS_TOKEN"@github.com/datachain-ai/studio.git "$BRANCH" | grep -F "$BRANCH" 2>&1>/dev/null
          then
              STUDIO_BRANCH="$BRANCH"
          else
              STUDIO_BRANCH=main
          fi
          echo "STUDIO_BRANCH=$STUDIO_BRANCH" >> $GITHUB_ENV
          echo "Studio branch: $STUDIO_BRANCH"
 
      - name: Check out Studio
        uses: actions/checkout@v7
        with:
          fetch-depth: 1
          repository: datachain-ai/studio
          ref: ${{ env.STUDIO_BRANCH }}
          token: ${{ secrets.ITERATIVE_STUDIO_READ_ACCESS_TOKEN }}
 
      - name: Check out DataChain
        uses: actions/checkout@v7
        with:
          path: './datachain'
          fetch-depth: 1
 
      - name: Set up Python ${{ matrix.pyv }}
        uses: actions/setup-python@v6
        with:
          cache: 'pip'
          python-version: ${{ matrix.pyv }}
 
      - name: Setup uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-suffix: studio
          cache-dependency-glob: |
            datachain_saas/pyproject.toml
            datachain/pyproject.toml
 
      - name: Use CPU-only PyTorch on Linux
        # torch 2.11+ on PyPI pulls CUDA deps missing libnppicc (nvidia-npp)
        run: echo "UV_TORCH_BACKEND=cpu" >> "$GITHUB_ENV"
 
      - name: Update DataChain requirement in Studio
        if: ${{ env.BRANCH != 'main' }}
        working-directory: backend
        run: make update_datachain_deps "${{ env.BRANCH }}"
 
      - name: Install FFmpeg
        run: |
          sudo apt update
          sudo apt install -y ffmpeg
 
      - name: Install dependencies
        run: uv pip install --system ./datachain_saas[tests] ./datachain[tests]
 
      - name: Print installed system packages
        run: uv pip list --system
 
      - name: Initialize datachain venv
        env:
          PYTHON_VERSION: ${{ matrix.pyv }}
          DATACHAIN_VENV_DIR: /tmp/local/datachain_venv/python${{ matrix.pyv }}
        run: |
          virtualenv -p "$(which python"${PYTHON_VERSION}")" "${DATACHAIN_VENV_DIR}"
 
          pip_cache_dir="${DATACHAIN_VENV_DIR}/.cache/pip"
          pip_wheel_dir="${pip_cache_dir}/wheels"
          pip_bin="${DATACHAIN_VENV_DIR}/bin/pip"
          mkdir -p "$pip_cache_dir"
          mkdir -p "$pip_wheel_dir"
 
          uv_cache_dir="${DATACHAIN_VENV_DIR}/.cache/uv"
          mkdir -p "$uv_cache_dir"
 
          $pip_bin install -U pip wheel setuptools \
            --cache-dir="$pip_cache_dir"
 
          $pip_bin wheel ./datachain_saas \
            --wheel-dir="$pip_wheel_dir" \
            --cache-dir="$pip_cache_dir"
 
          uv venv --python "$PYTHON_VERSION" "${DATACHAIN_VENV_DIR}/default"
          uv pip install -r ./compute/requirements-worker-venv.txt \
            --find-links="$pip_wheel_dir" \
            --cache-dir="${DATACHAIN_VENV_DIR}/.cache/uv" \
            -p "${DATACHAIN_VENV_DIR}/default/bin/python"
 
      - name: Print installed worker venv packages
        env:
          DATACHAIN_VENV_DIR: /tmp/local/datachain_venv/python${{ matrix.pyv }}
        run: uv pip list -p "${DATACHAIN_VENV_DIR}/default/bin/python"
 
      - name: Run tests
        # Generate `.test_durations` file with `pytest --store-durations --durations-path ../.github/.test_durations ...`
        run: >
          PYTHONPATH="$(pwd)/..:${PYTHONPATH}"
          pytest
          --config-file=pyproject.toml -rs
          --splits=6 --group=${{ matrix.group }} --durations-path=../../.github/.test_durations
          --benchmark-skip
          tests ../datachain/tests
        working-directory: datachain_saas