bench-compare.yaml

name: Benchmark Pull Requests

on:
  pull_request:

env:
  BENCH_HISTORY_BRANCH: benchmarks-history
  BENCH_HISTORY_DIR: tmp-bench-history

jobs:
  bench-group:
    name: Benchmark Group (${{ matrix.group }})
    runs-on: ubuntu-latest
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        include:
          - group: indexing
            criterion_target: indexing
          - group: large_dataset
            criterion_target: large_dataset
          - group: sampling
            criterion_target: sampling
          - group: counts
            criterion_target: counts
          - group: algorithms
            criterion_target: algorithms
          - group: sample_entity_scaling
            criterion_target: sample_entity_scaling
          - group: examples
            criterion_target: examples

    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Setup environment
        uses: ./.github/actions/setup-env

      - name: Run criterion benchmarks for group
        id: bench
        run: |
          set -uo pipefail

          group="${{ matrix.group }}"
          target="${{ matrix.criterion_target }}"

          base_status=0
          compare_status=0
          regress_status=0

          git checkout --force "origin/${{ github.event.pull_request.base.ref }}"
          cargo bench -p ixa-bench --bench "$target" -- --save-baseline base 2>&1 | tee "criterion-base-${group}.txt" || base_status=$?

          git checkout --force "${{ github.event.pull_request.head.sha }}"
          if cargo bench -p ixa-bench --bench "$target" -- --baseline base 2>&1 | tee "criterion-compare-${group}.txt"; then
            cargo run -q -p ixa-bench --bin check_criterion_regressions 2>&1 | tee "criterion-regressions-${group}.txt" || regress_status=$?
          else
            compare_status=$?
            echo "Note: A comparison could not be generated. Maybe you added new benchmarks?" | tee "criterion-regressions-${group}.txt"
            cargo bench -p ixa-bench --bench "$target" 2>&1 | tee -a "criterion-compare-${group}.txt" || true
          fi

          {
            echo "group=${group}"
            echo "target=${target}"
            echo "base_status=${base_status}"
            echo "compare_status=${compare_status}"
            echo "regress_status=${regress_status}"
            echo "base_ref=${{ github.event.pull_request.base.ref }}"
            echo "base_sha=${{ github.event.pull_request.base.sha }}"
            echo "head_ref=${{ github.event.pull_request.head.ref }}"
            echo "head_sha=${{ github.event.pull_request.head.sha }}"
          } > "status-${group}.env"

          if [ "$base_status" -ne 0 ] || [ "$compare_status" -ne 0 ] || [ "$regress_status" -ne 0 ]; then
            exit 1
          fi

      - name: Format group markdown
        if: ${{ always() }}
        run: |
          set -euo pipefail

          group="${{ matrix.group }}"

          if [ -f "status-${group}.env" ]; then
            # shellcheck disable=SC1090
            source "status-${group}.env"
          else
            base_status="unknown"
            compare_status="unknown"
            regress_status="unknown"
            base_ref="${{ github.event.pull_request.base.ref }}"
            base_sha="${{ github.event.pull_request.base.sha }}"
            head_ref="${{ github.event.pull_request.head.ref }}"
            head_sha="${{ github.event.pull_request.head.sha }}"
          fi

          {
            echo "## ${group}"
            echo
            printf -- '- Base: `%s` (`%s`)\n' "$base_ref" "$base_sha"
            printf -- '- Head: `%s` (`%s`)\n' "$head_ref" "$head_sha"
            printf -- '- Statuses: baseline=`%s`, compare=`%s`, regressions=`%s`\n' "$base_status" "$compare_status" "$regress_status"
            echo
            echo '```'
            if [ -f "criterion-regressions-${group}.txt" ]; then
              cat "criterion-regressions-${group}.txt"
            else
              echo "No regression output was generated for this group."
            fi
            echo '```'
            echo
          } > "results_${group}.md"

      - name: Upload group markdown artifact
        if: ${{ always() }}
        uses: actions/upload-artifact@v7
        with:
          name: results-${{ matrix.group }}
          path: results_${{ matrix.group }}.md

      - name: Upload group raw logs artifact
        if: ${{ always() }}
        uses: actions/upload-artifact@v7
        with:
          name: raw-${{ matrix.group }}
          if-no-files-found: ignore
          path: |
            criterion-base-${{ matrix.group }}.txt
            criterion-compare-${{ matrix.group }}.txt
            criterion-regressions-${{ matrix.group }}.txt
            status-${{ matrix.group }}.env

  hyperfine:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Setup environment
        uses: ./.github/actions/setup-env

      - name: Run hyperfine benchmarks
        run: |
          set -euo pipefail
          mise run bench:hyperfine -- -- --export-markdown hyperfine.md --export-json hyperfine.json

      - name: Upload hyperfine artifact
        uses: actions/upload-artifact@v7
        with:
          name: hyperfine
          path: |
            hyperfine.md
            hyperfine.json

  aggregate-results:
    runs-on: ubuntu-latest
    needs:
      - bench-group
      - hyperfine
    if: ${{ always() }}
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Setup environment
        uses: ./.github/actions/setup-env

      - name: Download group markdown artifacts
        uses: actions/download-artifact@v8
        with:
          pattern: results-*
          merge-multiple: true
          path: artifacts/results

      - name: Download group raw log artifacts
        uses: actions/download-artifact@v8
        with:
          pattern: raw-*
          merge-multiple: true
          path: artifacts/raw

      - name: Download hyperfine artifact
        uses: actions/download-artifact@v8
        with:
          name: hyperfine

      - name: Build combined PR comment markdown
        run: |
          set -euo pipefail

          groups=(
            indexing
            large_dataset
            sampling
            counts
            algorithms
            sample_entity_scaling
            examples
          )

          node scripts/format_bench_pr_comment.mjs \
            --out results.md \
            --hyperfine-md hyperfine.md \
            --criterion-dir artifacts/raw \
            --groups "$(IFS=,; echo "${groups[*]}")"

      - name: Upload PR comment artifact
        uses: actions/upload-artifact@v7
        with:
          name: pr-comment
          path: results.md

      - name: Skip bench history JSON (PR not targeting main)
        if: ${{ github.event.pull_request.base.ref != 'main' }}
        run: |
          set -euo pipefail
          echo "PR targets '${{ github.event.pull_request.base.ref }}', not 'main'; skipping bench history JSON steps."
          {
            echo "### Bench history"
            echo "Skipping bench history JSON generation/publish because this PR targets '${{ github.event.pull_request.base.ref }}' (not 'main')."
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Download previous bench history from benchmarks-history branch (best effort)
        if: ${{ github.event.pull_request.base.ref == 'main' }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -euo pipefail
          if gh api \
            -H "Accept: application/vnd.github+json" \
            "/repos/${GITHUB_REPOSITORY}/contents/bench-history.json?ref=${BENCH_HISTORY_BRANCH}" \
            --jq '.content' > bench-history.b64 2>/dev/null; then
            base64 -d bench-history.b64 > bench-history.json
            echo "Downloaded bench-history.json from ${BENCH_HISTORY_BRANCH}"
          else
            echo "No bench-history.json found on ${BENCH_HISTORY_BRANCH} (first run?)."
          fi

      - name: Merge criterion group logs
        if: ${{ github.event.pull_request.base.ref == 'main' }}
        run: |
          set -euo pipefail

          groups=(
            indexing
            large_dataset
            sampling
            counts
            algorithms
            sample_entity_scaling
            examples
          )

          : > criterion-compare.txt
          for group in "${groups[@]}"; do
            file="artifacts/raw/criterion-compare-${group}.txt"
            if [ -f "$file" ]; then
              cat "$file" >> criterion-compare.txt
              echo '' >> criterion-compare.txt
            fi
          done

      - name: Create JSON results
        if: ${{ github.event.pull_request.base.ref == 'main' }}
        env:
          PR_NUMBER: ${{ github.event.pull_request.number || '' }}
          BASE_REF: ${{ github.event.pull_request.base.ref || '' }}
          BASE_SHA: ${{ github.event.pull_request.base.sha || '' }}
          HEAD_REF: ${{ github.event.pull_request.head.ref || github.ref_name }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
          RUN_BRANCH: ${{ github.event.pull_request.head.ref || github.ref_name }}
        run: |
          set -euo pipefail
          node scripts/bench_results.mjs \
            --repo "${GITHUB_REPOSITORY}" \
            --branch "${RUN_BRANCH}" \
            --pr-number "${PR_NUMBER}" \
            --base-ref "${BASE_REF}" --base-sha "${BASE_SHA}" \
            --head-ref "${HEAD_REF}" --head-sha "${HEAD_SHA}" \
            --hyperfine-json hyperfine.json \
            --criterion-log criterion-compare.txt \
            --history-in bench-history.json \
            --out-current bench-current.json \
            --history-out bench-history.json

      - name: Upload bench history artifact
        if: ${{ github.event.pull_request.base.ref == 'main' }}
        uses: actions/upload-artifact@v7
        with:
          name: bench-history
          path: bench-history.json

  comment-on-pr:
    runs-on: ubuntu-latest
    needs: aggregate-results
    permissions:
      pull-requests: write
    steps:
      - name: Download PR comment artifact
        uses: actions/download-artifact@v8
        with:
          name: pr-comment

      - name: Add comment to PR
        run: |
          gh pr comment ${{ github.event.pull_request.number }} \
            --repo ${{ github.repository }} \
            --body-file results.md
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  publish-bench-history:
    runs-on: ubuntu-latest
    needs: aggregate-results
    if: ${{ github.event.pull_request.base.ref == 'main' }}
    permissions:
      contents: write
    steps:
      - name: Download bench history artifact
        uses: actions/download-artifact@v8
        with:
          name: bench-history

      - name: Checkout benchmarks-history branch
        uses: actions/checkout@v6
        with:
          ref: ${{ env.BENCH_HISTORY_BRANCH }}
          path: ${{ env.BENCH_HISTORY_DIR }}
          fetch-depth: 1
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Publish bench-history.json to benchmarks-history branch
        env:
          PR_NUMBER: ${{ github.event.pull_request.number }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          set -euo pipefail
          cp bench-history.json "${BENCH_HISTORY_DIR}/bench-history.json"
          pushd "${BENCH_HISTORY_DIR}" >/dev/null
          git checkout "${BENCH_HISTORY_BRANCH}"
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          if [ -z "$(git status --porcelain)" ]; then
            echo "No changes to bench-history.json"
            exit 0
          fi
          git add bench-history.json
          git commit -m "Update bench-history.json (PR #${PR_NUMBER} @ ${HEAD_SHA:0:7})"
          git push
          popd >/dev/null