Merge pull request #585 from mlcommons/mlperf-inference

Sync <- Mlperf inference for November release

Author: arjunsuresh

.github/workflows/test-cm-based-submission-generation.yml

@@ -19,7 +19,7 @@ jobs:
         python-version: [ "3.12" ]
         division: ["closed", "open", "closed-open"]
         category: ["datacenter", "edge"]
-        case: ["case-3", "case-7", "case-8"]
+        case: ["closed", "closed-no-compliance", "closed-power", "closed-failed-power-logs", "case-3", "case-7", "case-8"]
         action: ["run", "docker"]
         exclude:
           - os: macos-latest
@@ -37,21 +37,31 @@ jobs:
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
     - name: Pull repo where test cases are uploaded
       run: |
-        git clone -b submission-generation-tests https://github.com/anandhu-eng/inference.git submission_generation_tests
+        git clone -b submission-generation-tests https://github.com/mlcommons/inference.git submission_generation_tests
     - name: Run Submission Generation - ${{ matrix.case }} ${{ matrix.action }} ${{ matrix.category }} ${{ matrix.division }} 
       run: |
         if [ "${{ matrix.case }}" == "case-3" ]; then
-          #results_dir="submission_generation_tests/case-3/"
           description="Submission generation (model_mapping.json not present but model name matches with official one)"
         elif [ "${{ matrix.case }}" == "case-7" ]; then
-          #results_dir="submission_generation_tests/case-7/"
           description="Submission generation (sut_info.json incomplete, SUT folder name in required format)"
         elif [ "${{ matrix.case }}" == "case-8" ]; then
-          #results_dir="submission_generation_tests/case-8/"
+          extra_run_args=" --category=datacenter"
           description="Submission generation (system_meta.json not found in results folder)"
+        elif [ "${{ matrix.case }}" == "closed" ]; then
+          extra_run_args=" --env.CM_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check""
+          description="Test submission - contains closed edge and datacenter"
+        elif [ "${{ matrix.case }}" == "closed-no-compliance" ]; then
+          extra_run_args=" --env.CM_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check""
+          description="Test submission - contains closed edge and datacenter with no compliance tests"
+        elif [ "${{ matrix.case }}" == "closed-power" ]; then
+          extra_run_args=" --env.CM_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check""
+          description="Test submission - contains closed-power edge and datacenter results"
+        elif [ "${{ matrix.case }}" == "closed-failed-power-logs" ]; then
+          extra_run_args=" --env.CM_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check""
+          description="Test submission - contains closed-power edge and datacenter results with failed power logs"
         fi
         # Dynamically set the log group to simulate a dynamic step name
         echo "::group::$description"
-        cm ${{ matrix.action }} script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet
+        cm ${{ matrix.action }} script --tags=generate,inference,submission --adr.submission-checker-src.tags=_branch.dev --clean --preprocess_submission=yes --results_dir=$PWD/submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $extra_run_args
         echo "::endgroup::"
   

.github/workflows/test-mlperf-inference-mixtral.yml

@@ -30,5 +30,5 @@ jobs:
         git config --global credential.helper store
         huggingface-cli login --token ${{ secrets.HF_TOKEN }} --add-to-git-credential
         cm pull repo
-        cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=mixtral-8x7b --implementation=reference --batch_size=1 --precision=${{ matrix.precision }} --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=1 --target_qps=0.001 --clean --env.CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --env.CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes
+        cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=mixtral-8x7b --implementation=reference --batch_size=1 --precision=${{ matrix.precision }} --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=3 --target_qps=0.001 --clean --env.CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --env.CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --adr.openorca-mbxp-gsm8k-combined-preprocessed.tags=_generate-test-data.1
         cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - GO-phoenix" --quiet --submission_dir=$HOME/gh_action_submissions

.github/workflows/test-nvidia-mlperf-inference-implementations.yml

@@ -39,6 +39,6 @@ jobs:
           pip install --upgrade cm4mlops
           cm pull repo
           
-          cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --adr.submission-checker-src.tags=_branch.dev --execution_mode=valid --gpu_name=rtx_4090 --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=$hw_name --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed  --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean  --docker --quiet
+          cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --adr.submission-checker-src.tags=_branch.dev --execution_mode=valid --gpu_name=rtx_4090 --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=$hw_name --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed  --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_model_from_host=yes --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean  --docker --quiet
 
           cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on NVIDIA_$hw_name" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=$hw_name

VERSION

@@ -1 +1 @@
-0.3.26
+0.4.0

script/activate-python-venv/_cm.json

@@ -0,0 +1,18 @@
+alias: activate-python-venv
+automation_alias: script
+automation_uid: 5b4e0237da074764
+category: Python automation
+developers: '[Grigori Fursin](https://cKnowledge.org/gfursin)'
+name: Activate virtual Python environment
+prehook_deps:
+- names:
+  - python-venv
+  reuse_version: true
+  tags: install,python-venv
+tags:
+- activate
+- python
+- activate-python-venv
+- python-venv
+tags_help: activate python-venv
+uid: fcbbb84946f34c55

script/activate-python-venv/_cm.yaml

@@ -0,0 +1,27 @@
+alias: app-image-classification-tf-onnx-cpp
+automation_alias: script
+automation_uid: 5b4e0237da074764
+category: Modular AI/ML application pipeline
+default_env:
+  CM_BATCH_COUNT: '1'
+  CM_BATCH_SIZE: '1'
+deps:
+- tags: detect,os
+- tags: get,sys-utils-cm
+- tags: get,gcc
+- tags: get,dataset,image-classification,original
+- tags: get,dataset-aux,image-classification
+- tags: get,ml-model,raw,image-classification,resnet50,_onnx,_opset-11
+- tags: tensorflow,from-src
+  version: v2.0.0
+tags:
+- app
+- image-classification
+- tf
+- tensorflow
+- tf-onnx
+- tensorflow-onnx
+- onnx
+- cpp
+tags_help: app image-classification cpp tensorflow onnx
+uid: 879ed32e47074033

script/app-image-classification-tf-onnx-cpp/_cm.json

@@ -0,0 +1,46 @@
+alias: app-image-classification-torch-py
+automation_alias: script
+automation_uid: 5b4e0237da074764
+category: Modular AI/ML application pipeline
+default_env:
+  CM_BATCH_COUNT: '1'
+  CM_BATCH_SIZE: '1'
+deps:
+- tags: detect,os
+- names:
+  - python
+  - python3
+  tags: get,python3
+- tags: get,dataset,imagenet,image-classification,preprocessed
+- tags: get,dataset-aux,imagenet-aux,image-classification
+- tags: get,imagenet-helper
+- tags: get,ml-model,image-classification,resnet50,_pytorch,_fp32
+- skip_if_env:
+    USE_CUDA:
+    - 'yes'
+  tags: get,generic-python-lib,_torch
+- enable_if_env:
+    USE_CUDA:
+    - 'yes'
+  tags: get,generic-python-lib,_torch_cuda
+- skip_if_env:
+    USE_CUDA:
+    - 'yes'
+  tags: get,generic-python-lib,_torchvision
+- enable_if_env:
+    USE_CUDA:
+    - 'yes'
+  tags: get,generic-python-lib,_torchvision_cuda
+tags:
+- app
+- image-classification
+- torch
+- python
+tags_help: app image-classification python torch
+uid: e3986ae887b84ca8
+variations:
+  cuda:
+    deps:
+    - tags: get,cuda
+    env:
+      USE_CUDA: 'yes'