CI workflow (MotleyAI/slayer)

name: CI
 
permissions:
  contents: read
 
on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
 
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
 
jobs:
  lint-and-test:
    timeout-minutes: 30
    runs-on: latchkey-small
    strategy:
      matrix:
        python-version: ["3.11", "3.14"]
 
    steps:
      - uses: actions/checkout@v4
 
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          cache: 'pip'
          python-version: ${{ matrix.python-version }}
 
      - name: Set up JDK 17 (for Flight SQL JDBC integration tests)
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: "17"
 
      - name: Install Poetry
        run: pip install poetry
 
      - name: Install dependencies
        run: poetry install -E all --with dev
 
      - name: Install jafgen (for notebook tests, not on PyPI)
        run: poetry run pip install git+https://github.com/rossbowen/jaffle-shop-generator.git@09557a1118b000071f8171aa97d54d5029bf0f0b
 
      - name: Lint with ruff
        run: poetry run ruff check slayer/ tests/
 
      - name: Run unit tests
        timeout-minutes: 10
        run: poetry run pytest tests/ -v -m "not integration" --timeout=120
 
      - name: Run integration tests
        timeout-minutes: 20
        # DEV-1564: the MySQL / ClickHouse / SQL Server pytest integration
        # suites have their own path-gated workflows under
        # .github/workflows/integration-<dialect>.yml. They require Docker
        # + testcontainers + (for SQL Server) the ODBC Driver 18 - provisioned
        # only in those workflows. Exclude them from the always-on integration
        # job; they run when their respective dialect files change.
        #
        # DEV-1562: the live-Metabase e2e suite (`metabase_e2e` marker) has
        # its own workflow at .github/workflows/pg-facade-e2e.yml - same
        # reasoning, exclude here so this job isn't on the hook for booting
        # a Metabase container.
        run: |
          poetry run pytest tests/ -v -m "integration and not metabase_e2e" --timeout=180 \
            --ignore=tests/integration/test_integration_mysql.py \
            --ignore=tests/integration/test_integration_clickhouse.py \
            --ignore=tests/integration/test_integration_sqlserver.py
 
  # DEV-1564: the previous `integration-examples` matrix (verify.py-based
  # end-to-end checks for ClickHouse + MySQL) migrated into the per-dialect
  # workflows so each dialect's CI lives in one place and is path-gated.
  # SQL Server gained its first verify.py-based CI coverage there too.
 
  bigquery-example:
    timeout-minutes: 30
    # Skip on forked-PR runs where the GCP secrets aren't available - the job
    # would fail otherwise and we'd rather not nag external contributors.
    if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
    runs-on: latchkey-small
    needs: lint-and-test
    steps:
      - uses: actions/checkout@v4
 
      - name: Skip if GCP secrets are unset
        id: gate
        env:
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
          GCP_SA_KEY_B64: ${{ secrets.GCP_SA_KEY_B64 }}
        run: |
          if [[ -z "$GCP_PROJECT_ID" || -z "$GCP_SA_KEY_B64" ]]; then
            echo "BigQuery secrets unset - skipping the example."
            echo "skip=true" >> "$GITHUB_OUTPUT"
          else
            echo "skip=false" >> "$GITHUB_OUTPUT"
          fi
 
      - name: Set up Python
        if: steps.gate.outputs.skip != 'true'
        uses: actions/setup-python@v5
        with:
          cache: 'pip'
          python-version: "3.11"
 
      - name: Install Poetry
        if: steps.gate.outputs.skip != 'true'
        run: pip install poetry
 
      - name: Install dependencies (with bigquery extra)
        if: steps.gate.outputs.skip != 'true'
        run: poetry install -E bigquery
 
      - name: Decode service-account JSON key
        if: steps.gate.outputs.skip != 'true'
        env:
          GCP_SA_KEY_B64: ${{ secrets.GCP_SA_KEY_B64 }}
        run: |
          echo "$GCP_SA_KEY_B64" | base64 -d > "$RUNNER_TEMP/sa-key.json"
          echo "GOOGLE_APPLICATION_CREDENTIALS=$RUNNER_TEMP/sa-key.json" >> "$GITHUB_ENV"
 
      - name: Start SLayer server
        if: steps.gate.outputs.skip != 'true'
        env:
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
        run: |
          poetry run bash examples/bigquery/start.sh > /tmp/slayer-bq.log 2>&1 &
          echo "SLAYER_PID=$!" >> "$GITHUB_ENV"
 
      - name: Wait for SLayer API to accept connections
        if: steps.gate.outputs.skip != 'true'
        run: |
          for i in $(seq 1 60); do
            if curl -sf http://localhost:5143/health >/dev/null; then
              echo "SLayer API ready after ${i} attempts"
              exit 0
            fi
            sleep 2
          done
          echo "SLayer API never came up - logs:" >&2
          cat /tmp/slayer-bq.log
          exit 1
 
      - name: Run verify.py
        if: steps.gate.outputs.skip != 'true'
        timeout-minutes: 5
        run: poetry run python examples/bigquery/verify.py
 
      - name: Dump server logs on failure
        if: failure() && steps.gate.outputs.skip != 'true'
        run: cat /tmp/slayer-bq.log
 
      - name: Stop SLayer server
        if: always() && steps.gate.outputs.skip != 'true'
        run: |
          if [ -n "${SLAYER_PID:-}" ]; then
            kill "$SLAYER_PID" 2>/dev/null || true
          fi
          rm -f "$RUNNER_TEMP/sa-key.json"
 
  snowflake-integration:
    timeout-minutes: 30
    # Skip on forked-PR runs where the Snowflake secret isn't available - the
    # job would fail otherwise and we'd rather not nag external contributors.
    if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
    runs-on: latchkey-small
    needs: lint-and-test
    steps:
      - uses: actions/checkout@v4
 
      - name: Skip if Snowflake secret is unset
        id: gate
        env:
          SNOWFLAKE_CONNECTIONS_TOML: ${{ secrets.SNOWFLAKE_CONNECTIONS_TOML }}
        run: |
          if [[ -z "$SNOWFLAKE_CONNECTIONS_TOML" ]]; then
            echo "Snowflake secret unset - skipping the integration suite."
            echo "skip=true" >> "$GITHUB_OUTPUT"
          else
            echo "skip=false" >> "$GITHUB_OUTPUT"
          fi
 
      - name: Set up Python
        if: steps.gate.outputs.skip != 'true'
        uses: actions/setup-python@v5
        with:
          cache: 'pip'
          python-version: "3.11"
 
      - name: Install Poetry
        if: steps.gate.outputs.skip != 'true'
        run: pip install poetry
 
      - name: Install dependencies (with snowflake extra)
        if: steps.gate.outputs.skip != 'true'
        run: poetry install -E snowflake
 
      - name: Write ~/.snowflake/connections.toml from secret
        if: steps.gate.outputs.skip != 'true'
        env:
          SNOWFLAKE_CONNECTIONS_TOML: ${{ secrets.SNOWFLAKE_CONNECTIONS_TOML }}
        run: |
          mkdir -p ~/.snowflake
          # `printf '%s'` is literal - no backslash mangling, no extra trailing
          # newline. `chmod 600` is required: snowflake-connector-python
          # refuses to load a TOML that's world-/group-readable.
          printf '%s' "$SNOWFLAKE_CONNECTIONS_TOML" > ~/.snowflake/connections.toml
          chmod 600 ~/.snowflake/connections.toml
 
      - name: Run Snowflake integration tests
        if: steps.gate.outputs.skip != 'true'
        timeout-minutes: 15
        # SLAYER_SNOWFLAKE_CONNECTION names the profile in the TOML the
        # integration suite should use. The default in the test file is
        # `slayer_test` - match the section header in
        # `secrets.SNOWFLAKE_CONNECTIONS_TOML`.
        env:
          SLAYER_SNOWFLAKE_CONNECTION: slayer_test
        run: poetry run pytest tests/integration/test_integration_snowflake.py -v -m integration --timeout=300
 
      - name: Wipe Snowflake credentials
        if: always()
        run: rm -f ~/.snowflake/connections.toml