chore: remove tuple queries #701
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark Pull Requests | |
| on: | |
| pull_request: | |
| env: | |
| BENCH_HISTORY_BRANCH: benchmarks-history | |
| BENCH_HISTORY_DIR: tmp-bench-history | |
| jobs: | |
| bench-group: | |
| name: Benchmark Group (${{ matrix.group }}) | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - group: indexing | |
| criterion_target: indexing | |
| - group: large_dataset | |
| criterion_target: large_dataset | |
| - group: sampling | |
| criterion_target: sampling | |
| - group: counts | |
| criterion_target: counts | |
| - group: algorithms | |
| criterion_target: algorithms | |
| - group: sample_entity_scaling | |
| criterion_target: sample_entity_scaling | |
| - group: examples | |
| criterion_target: examples | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup environment | |
| uses: ./.github/actions/setup-env | |
| - name: Run criterion benchmarks for group | |
| id: bench | |
| run: | | |
| set -uo pipefail | |
| group="${{ matrix.group }}" | |
| target="${{ matrix.criterion_target }}" | |
| base_status=0 | |
| compare_status=0 | |
| regress_status=0 | |
| git checkout --force "origin/${{ github.event.pull_request.base.ref }}" | |
| cargo bench -p ixa-bench --bench "$target" -- --save-baseline base 2>&1 | tee "criterion-base-${group}.txt" || base_status=$? | |
| git checkout --force "${{ github.event.pull_request.head.sha }}" | |
| if cargo bench -p ixa-bench --bench "$target" -- --baseline base 2>&1 | tee "criterion-compare-${group}.txt"; then | |
| cargo run -q -p ixa-bench --bin check_criterion_regressions 2>&1 | tee "criterion-regressions-${group}.txt" || regress_status=$? | |
| else | |
| compare_status=$? | |
| echo "Note: A comparison could not be generated. Maybe you added new benchmarks?" | tee "criterion-regressions-${group}.txt" | |
| cargo bench -p ixa-bench --bench "$target" 2>&1 | tee -a "criterion-compare-${group}.txt" || true | |
| fi | |
| { | |
| echo "group=${group}" | |
| echo "target=${target}" | |
| echo "base_status=${base_status}" | |
| echo "compare_status=${compare_status}" | |
| echo "regress_status=${regress_status}" | |
| echo "base_ref=${{ github.event.pull_request.base.ref }}" | |
| echo "base_sha=${{ github.event.pull_request.base.sha }}" | |
| echo "head_ref=${{ github.event.pull_request.head.ref }}" | |
| echo "head_sha=${{ github.event.pull_request.head.sha }}" | |
| } > "status-${group}.env" | |
| if [ "$base_status" -ne 0 ] || [ "$compare_status" -ne 0 ] || [ "$regress_status" -ne 0 ]; then | |
| exit 1 | |
| fi | |
| - name: Format group markdown | |
| if: ${{ always() }} | |
| run: | | |
| set -euo pipefail | |
| group="${{ matrix.group }}" | |
| if [ -f "status-${group}.env" ]; then | |
| # shellcheck disable=SC1090 | |
| source "status-${group}.env" | |
| else | |
| base_status="unknown" | |
| compare_status="unknown" | |
| regress_status="unknown" | |
| base_ref="${{ github.event.pull_request.base.ref }}" | |
| base_sha="${{ github.event.pull_request.base.sha }}" | |
| head_ref="${{ github.event.pull_request.head.ref }}" | |
| head_sha="${{ github.event.pull_request.head.sha }}" | |
| fi | |
| { | |
| echo "## ${group}" | |
| echo | |
| printf -- '- Base: `%s` (`%s`)\n' "$base_ref" "$base_sha" | |
| printf -- '- Head: `%s` (`%s`)\n' "$head_ref" "$head_sha" | |
| printf -- '- Statuses: baseline=`%s`, compare=`%s`, regressions=`%s`\n' "$base_status" "$compare_status" "$regress_status" | |
| echo | |
| echo '```' | |
| if [ -f "criterion-regressions-${group}.txt" ]; then | |
| cat "criterion-regressions-${group}.txt" | |
| else | |
| echo "No regression output was generated for this group." | |
| fi | |
| echo '```' | |
| echo | |
| } > "results_${group}.md" | |
| - name: Upload group markdown artifact | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: results-${{ matrix.group }} | |
| path: results_${{ matrix.group }}.md | |
| - name: Upload group raw logs artifact | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: raw-${{ matrix.group }} | |
| if-no-files-found: ignore | |
| path: | | |
| criterion-base-${{ matrix.group }}.txt | |
| criterion-compare-${{ matrix.group }}.txt | |
| criterion-regressions-${{ matrix.group }}.txt | |
| status-${{ matrix.group }}.env | |
| hyperfine: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup environment | |
| uses: ./.github/actions/setup-env | |
| - name: Run hyperfine benchmarks | |
| run: | | |
| set -euo pipefail | |
| mise run bench:hyperfine -- -- --export-markdown hyperfine.md --export-json hyperfine.json | |
| - name: Upload hyperfine artifact | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: hyperfine | |
| path: | | |
| hyperfine.md | |
| hyperfine.json | |
| aggregate-results: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - bench-group | |
| - hyperfine | |
| if: ${{ always() }} | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup environment | |
| uses: ./.github/actions/setup-env | |
| - name: Download group markdown artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: results-* | |
| merge-multiple: true | |
| path: artifacts/results | |
| - name: Download group raw log artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: raw-* | |
| merge-multiple: true | |
| path: artifacts/raw | |
| - name: Download hyperfine artifact | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: hyperfine | |
| - name: Build combined PR comment markdown | |
| run: | | |
| set -euo pipefail | |
| groups=( | |
| indexing | |
| large_dataset | |
| sampling | |
| counts | |
| algorithms | |
| sample_entity_scaling | |
| examples | |
| ) | |
| node scripts/format_bench_pr_comment.mjs \ | |
| --out results.md \ | |
| --hyperfine-md hyperfine.md \ | |
| --criterion-dir artifacts/raw \ | |
| --groups "$(IFS=,; echo "${groups[*]}")" | |
| - name: Upload PR comment artifact | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: pr-comment | |
| path: results.md | |
| - name: Skip bench history JSON (PR not targeting main) | |
| if: ${{ github.event.pull_request.base.ref != 'main' }} | |
| run: | | |
| set -euo pipefail | |
| echo "PR targets '${{ github.event.pull_request.base.ref }}', not 'main'; skipping bench history JSON steps." | |
| { | |
| echo "### Bench history" | |
| echo "Skipping bench history JSON generation/publish because this PR targets '${{ github.event.pull_request.base.ref }}' (not 'main')." | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Download previous bench history from benchmarks-history branch (best effort) | |
| if: ${{ github.event.pull_request.base.ref == 'main' }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| if gh api \ | |
| -H "Accept: application/vnd.github+json" \ | |
| "/repos/${GITHUB_REPOSITORY}/contents/bench-history.json?ref=${BENCH_HISTORY_BRANCH}" \ | |
| --jq '.content' > bench-history.b64 2>/dev/null; then | |
| base64 -d bench-history.b64 > bench-history.json | |
| echo "Downloaded bench-history.json from ${BENCH_HISTORY_BRANCH}" | |
| else | |
| echo "No bench-history.json found on ${BENCH_HISTORY_BRANCH} (first run?)." | |
| fi | |
| - name: Merge criterion group logs | |
| if: ${{ github.event.pull_request.base.ref == 'main' }} | |
| run: | | |
| set -euo pipefail | |
| groups=( | |
| indexing | |
| large_dataset | |
| sampling | |
| counts | |
| algorithms | |
| sample_entity_scaling | |
| examples | |
| ) | |
| : > criterion-compare.txt | |
| for group in "${groups[@]}"; do | |
| file="artifacts/raw/criterion-compare-${group}.txt" | |
| if [ -f "$file" ]; then | |
| cat "$file" >> criterion-compare.txt | |
| echo '' >> criterion-compare.txt | |
| fi | |
| done | |
| - name: Create JSON results | |
| if: ${{ github.event.pull_request.base.ref == 'main' }} | |
| env: | |
| PR_NUMBER: ${{ github.event.pull_request.number || '' }} | |
| BASE_REF: ${{ github.event.pull_request.base.ref || '' }} | |
| BASE_SHA: ${{ github.event.pull_request.base.sha || '' }} | |
| HEAD_REF: ${{ github.event.pull_request.head.ref || github.ref_name }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} | |
| RUN_BRANCH: ${{ github.event.pull_request.head.ref || github.ref_name }} | |
| run: | | |
| set -euo pipefail | |
| node scripts/bench_results.mjs \ | |
| --repo "${GITHUB_REPOSITORY}" \ | |
| --branch "${RUN_BRANCH}" \ | |
| --pr-number "${PR_NUMBER}" \ | |
| --base-ref "${BASE_REF}" --base-sha "${BASE_SHA}" \ | |
| --head-ref "${HEAD_REF}" --head-sha "${HEAD_SHA}" \ | |
| --hyperfine-json hyperfine.json \ | |
| --criterion-log criterion-compare.txt \ | |
| --history-in bench-history.json \ | |
| --out-current bench-current.json \ | |
| --history-out bench-history.json | |
| - name: Upload bench history artifact | |
| if: ${{ github.event.pull_request.base.ref == 'main' }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: bench-history | |
| path: bench-history.json | |
| comment-on-pr: | |
| runs-on: ubuntu-latest | |
| needs: aggregate-results | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Download PR comment artifact | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: pr-comment | |
| - name: Add comment to PR | |
| run: | | |
| gh pr comment ${{ github.event.pull_request.number }} \ | |
| --repo ${{ github.repository }} \ | |
| --body-file results.md | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| publish-bench-history: | |
| runs-on: ubuntu-latest | |
| needs: aggregate-results | |
| if: ${{ github.event.pull_request.base.ref == 'main' }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Download bench history artifact | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: bench-history | |
| - name: Checkout benchmarks-history branch | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ env.BENCH_HISTORY_BRANCH }} | |
| path: ${{ env.BENCH_HISTORY_DIR }} | |
| fetch-depth: 1 | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Publish bench-history.json to benchmarks-history branch | |
| env: | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| set -euo pipefail | |
| cp bench-history.json "${BENCH_HISTORY_DIR}/bench-history.json" | |
| pushd "${BENCH_HISTORY_DIR}" >/dev/null | |
| git checkout "${BENCH_HISTORY_BRANCH}" | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| if [ -z "$(git status --porcelain)" ]; then | |
| echo "No changes to bench-history.json" | |
| exit 0 | |
| fi | |
| git add bench-history.json | |
| git commit -m "Update bench-history.json (PR #${PR_NUMBER} @ ${HEAD_SHA:0:7})" | |
| git push | |
| popd >/dev/null |