Skip to content

Commit fa16614

Browse files
arjunsuresharjunsuresh
authored
Merge pull request #557 from mlcommons/mlperf-inference
Sync <- Mlperf inference
2 parents 6521889 + 3a60122 commit fa16614

68 files changed

Lines changed: 1005 additions & 353 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/test-amd-mlperf-inference-implementations.yml‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ name: MLPerf Inference AMD implementations
22

33
on:
44
schedule:
5-
- cron: "29 4 * * *" #to be adjusted
5+
- cron: "46 11 * * *" #to be adjusted
66

77
jobs:
8-
build_nvidia:
9-
if: github.repository_owner == 'gateoverflow'
8+
run_amd:
9+
if: github.repository_owner == 'gateoverflow_off'
1010
runs-on: [ self-hosted, linux, x64, GO-spr ]
1111
strategy:
1212
fail-fast: false
@@ -16,11 +16,11 @@ jobs:
1616
steps:
1717
- name: Test MLPerf Inference AMD (build only) ${{ matrix.model }}
1818
run: |
19-
if [ -f "gh_action_conda/bin/deactivate" ]; then source gh_action_conda/bin/deactivate; fi
20-
python3 -m venv gh_action_conda
21-
source gh_action_conda/bin/activate
19+
if [ -f "gh_action/bin/deactivate" ]; then source gh_action/bin/deactivate; fi
20+
python3 -m venv gh_action
21+
source gh_action/bin/activate
2222
export CM_REPOS=$HOME/GH_CM
2323
pip install --upgrade cm4mlops
24-
pip install tabulate
25-
cm run script --tags=run-mlperf,inference,_all-scenarios,_full,_r4.1-dev --execution_mode=valid --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=IntelSPR.24c --implementation=amd --backend=pytorch --category=datacenter --division=open --scenario=Offline --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=rocm --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet
24+
cm pull repo
25+
cm run script --tags=run-mlperf,inference,_all-scenarios,_full,_r4.1-dev --execution_mode=valid --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=IntelSPR.24c --implementation=amd --backend=pytorch --category=datacenter --division=open --scenario=Offline --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=rocm --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet --docker_skip_run_cmd=yes
2626
# cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on SPR.24c" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=IntelSPR.24c

‎.github/workflows/test-cm-based-submission-generation.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,13 @@ jobs:
1717
matrix:
1818
os: [ubuntu-latest, windows-latest, macos-latest]
1919
python-version: [ "3.12" ]
20-
division: ["closed", "open"]
20+
division: ["closed", "open", "closed-open"]
2121
category: ["datacenter", "edge"]
22-
case: ["case-3", "case-7"]
22+
case: ["case-3", "case-7", "case-8"]
2323
action: ["run", "docker"]
2424
exclude:
2525
- os: macos-latest
2626
- os: windows-latest
27-
- division: "open"
2827
- category: "edge"
2928
steps:
3029
- uses: actions/checkout@v4
@@ -47,6 +46,9 @@ jobs:
4746
elif [ "${{ matrix.case }}" == "case-7" ]; then
4847
#results_dir="submission_generation_tests/case-7/"
4948
description="Submission generation (sut_info.json incomplete, SUT folder name in required format)"
49+
elif [ "${{ matrix.case }}" == "case-8" ]; then
50+
#results_dir="submission_generation_tests/case-8/"
51+
description="Submission generation (system_meta.json not found in results folder)"
5052
fi
5153
# Dynamically set the log group to simulate a dynamic step name
5254
echo "::group::$description"

‎.github/workflows/test-intel-mlperf-inference-implementations.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@ name: MLPerf Inference Intel implementations
22

33
on:
44
schedule:
5-
- cron: "29 1 * * *" #to be adjusted
5+
- cron: "29 1 * * 4" #to be adjusted
66

77
jobs:
8-
build_nvidia:
8+
run_intel:
99
if: github.repository_owner == 'gateoverflow'
1010
runs-on: [ self-hosted, linux, x64, GO-spr ]
1111
strategy:
@@ -22,5 +22,5 @@ jobs:
2222
export CM_REPOS=$HOME/GH_CM
2323
pip install --upgrade cm4mlops
2424
pip install tabulate
25-
cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --execution_mode=valid --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=IntelSPR.24c --implementation=intel --backend=pytorch --category=datacenter --division=open --scenario=Offline --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cpu --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet
25+
cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --execution_mode=valid --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=IntelSPR.24c --implementation=intel --backend=pytorch --category=datacenter --division=open --scenario=Offline --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cpu --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet
2626
cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on SPR.24c" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=IntelSPR.24c

‎.github/workflows/test-mlperf-inference-llama2.yml‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ name: MLPerf inference LLAMA 2 70B
55

66
on:
77
schedule:
8-
- cron: "30 2 * * 4"
8+
- cron: "59 04 * * *"
99

1010
jobs:
1111
build_reference:
@@ -17,9 +17,10 @@ jobs:
1717
python-version: [ "3.12" ]
1818
backend: [ "pytorch" ]
1919
device: [ "cpu" ]
20+
precision: [ "bfloat16" ]
2021

2122
steps:
22-
- name: Install dependencies
23+
- name: Test MLPerf Inference LLAMA 2 70B reference implementation
2324
run: |
2425
source gh_action/bin/deactivate || python3 -m venv gh_action
2526
source gh_action/bin/activate
@@ -28,7 +29,7 @@ jobs:
2829
pip install tabulate
2930
cm pull repo
3031
pip install "huggingface_hub[cli]"
32+
git config --global credential.helper store
3133
huggingface-cli login --token ${{ secrets.HF_TOKEN }} --add-to-git-credential
32-
- name: Test MLPerf Inference LLAMA 2 70B reference implementation
33-
run: |
34-
cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=llama2-70b-99 --implementation=reference --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --env.CM_MLPERF_MODEL_LLAMA2_70B_DOWNLOAD_TO_HOST=yes --adr.inference-src.tags=_repo.https://github.com/anandhu-eng/inference.git --clean
34+
cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=llama2-70b-99 --implementation=reference --backend=${{ matrix.backend }} --precision=${{ matrix.precision }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=0.001 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --env.CM_MLPERF_MODEL_LLAMA2_70B_DOWNLOAD_TO_HOST=yes --clean
35+
cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions" --quiet --submission_dir=$HOME/gh_action_submissions

‎.github/workflows/test-mlperf-inference-mixtral.yml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,19 @@ name: MLPerf inference MIXTRAL-8x7B
55

66
on:
77
schedule:
8-
- cron: "45 10 * * *" # 30th minute and 20th hour => 20:30 UTC => 2 AM IST
8+
- cron: "32 22 * * *" # 30th minute and 20th hour => 20:30 UTC => 2 AM IST
99

1010
jobs:
1111
build_reference:
1212
if: github.repository_owner == 'gateoverflow'
13-
runs-on: [ self-hosted, GO-spr, linux, x64 ]
13+
runs-on: [ self-hosted, phoenix, linux, x64 ]
1414
strategy:
1515
fail-fast: false
1616
matrix:
1717
python-version: [ "3.12" ]
1818
backend: [ "pytorch" ]
1919
device: [ "cpu" ]
20+
precision: [ "float16" ]
2021

2122
steps:
2223
- name: Test MLPerf Inference MIXTRAL-8X7B reference implementation
@@ -26,7 +27,8 @@ jobs:
2627
export CM_REPOS=$HOME/GH_CM
2728
pip install cm4mlops
2829
pip install "huggingface_hub[cli]"
30+
git config --global credential.helper store
2931
huggingface-cli login --token ${{ secrets.HF_TOKEN }} --add-to-git-credential
3032
cm pull repo
31-
cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=mixtral-8x7b --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=1 --target_qps=1 --clean --env.CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --env.CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes
32-
cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - GO-i9" --quiet --submission_dir=$HOME/gh_action_submissions
33+
cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=mixtral-8x7b --implementation=reference --batch_size=1 --precision=${{ matrix.precision }} --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=1 --target_qps=0.001 --clean --env.CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --env.CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes
34+
cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - GO-phoenix" --quiet --submission_dir=$HOME/gh_action_submissions

‎.github/workflows/test-mlperf-inference-sdxl.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: MLPerf inference SDXL
22
on:
33
schedule:
4-
- cron: "30 2 * * *"
4+
- cron: "19 17 * * *"
55

66
jobs:
77
build_reference:
@@ -21,5 +21,5 @@ jobs:
2121
export CM_REPOS=$HOME/GH_CM
2222
python3 -m pip install cm4mlops
2323
cm pull repo
24-
cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean
24+
cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --adr.mlperf-implementation.tags=_branch.dev --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean
2525
cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions

‎.github/workflows/test-nvidia-mlperf-inference-implementations.yml‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,43 @@ name: MLPerf Inference Nvidia implementations
22

33
on:
44
schedule:
5-
- cron: "49 19 * * *" #to be adjusted
5+
- cron: "54 22 * * *" #to be adjusted
66

77
jobs:
8-
build_nvidia:
8+
run_nvidia:
99
if: github.repository_owner == 'gateoverflow'
10-
runs-on: [ self-hosted, linux, x64, GO-spr ]
10+
runs-on:
11+
- self-hosted
12+
- linux
13+
- x64
14+
- cuda
15+
- ${{ matrix.system }}
1116
strategy:
1217
fail-fast: false
1318
matrix:
19+
system: [ "GO-spr", "phoenix", "i9" ]
1420
python-version: [ "3.12" ]
15-
model: [ "resnet50", "retinanet", "bert-99", "bert-99.9", "gptj-99.9", "3d-unet-99.9" ]
21+
model: [ "resnet50", "retinanet", "bert-99", "bert-99.9", "gptj-99.9", "3d-unet-99.9", "sdxl" ]
22+
exclude:
23+
- model: gptj-99.9
24+
1625
steps:
1726
- name: Test MLPerf Inference NVIDIA ${{ matrix.model }}
1827
run: |
28+
# Set hw_name based on matrix.system
29+
if [ "${{ matrix.system }}" = "GO-spr" ]; then
30+
hw_name="RTX4090x2"
31+
else
32+
hw_name="RTX4090x1"
33+
fi
34+
1935
if [ -f "gh_action/bin/deactivate" ]; then source gh_action/bin/deactivate; fi
2036
python3 -m venv gh_action
2137
source gh_action/bin/activate
2238
export CM_REPOS=$HOME/GH_CM
2339
pip install --upgrade cm4mlops
24-
pip install tabulate
25-
cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --execution_mode=valid --gpu_name=rtx_4090 --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=RTX4090x2 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet
26-
cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on NVIDIA_RTX4090x2" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=RTX4090x2
40+
cm pull repo
41+
42+
cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --adr.submission-checker-src.tags=_branch.dev --execution_mode=valid --gpu_name=rtx_4090 --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=$hw_name --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet
43+
44+
cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on NVIDIA_$hw_name" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=$hw_name

‎.github/workflows/test-scc24-sdxl.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: MLPerf inference SDXL (SCC)
22

33
on:
44
schedule:
5-
- cron: "35 19 * * *"
5+
- cron: "20 01 * * *"
66

77
jobs:
88
build_reference:
@@ -54,5 +54,5 @@ jobs:
5454
cm pull repo
5555
cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --pull_changes=yes --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --pull_changes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --hw_name=go-spr --custom_system_nvidia=yes --clean
5656
cm run script --tags=run-mlperf,inference,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean
57-
cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=datacenter --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/scc_gh_action_submissions --results_dir=$HOME/scc_gh_action_results/test_results
57+
cm run script --tags=generate,inference,submission --clean --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=datacenter --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/scc_gh_action_submissions --results_dir=$HOME/scc_gh_action_results/test_results
5858
cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/scc_gh_action_submissions

‎VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
0.3.25
1+
0.3.26

0 commit comments

Comments
 (0)