From 51e912386edde04111010af3fa29e16b4e582a08 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 2 Oct 2025 17:04:54 +0530 Subject: [PATCH 01/55] Skip ownership restore --- script/get-dataset-waymo/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-dataset-waymo/run.sh b/script/get-dataset-waymo/run.sh index 0af8d4a79..920224e60 100644 --- a/script/get-dataset-waymo/run.sh +++ b/script/get-dataset-waymo/run.sh @@ -9,6 +9,6 @@ if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" ]]; then cd "${MLC_DATASET_WAYMO_PATH}/training" || exit - for f in *.tar.gz; do tar -xzvf "$f"; done + for f in *.tar.gz; do tar --no-same-owner -xzvf "$f"; done cd - || exit -fi \ No newline at end of file +fi From 3c031d29e620ca1901d0ad6f0d2659afcd1d42d0 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Sat, 4 Oct 2025 11:22:59 +0530 Subject: [PATCH 02/55] support rgat model download via r2-downloader --- .../meta.yaml | 8 ++++--- script/app-mlperf-inference/meta.yaml | 6 +++++ script/get-ml-model-rgat/meta.yaml | 22 ++++++++++++++----- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index 3fdff095d..834020408 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -501,15 +501,17 @@ deps: - "on" ## RGAT - - tags: get,ml-model,rgat + - tags: get,ml-model,rgat,_r2-downloader,_mlcommons names: - rgat-model enable_if_env: MLC_MODEL: - rgat skip_if_env: - RGAT_CHECKPOINT_PATH: - - "on" + MLC_USE_MODEL_FROM_HOST: + - "yes" + MLC_RUN_STATE_DOCKER: + - "yes" ## LLAMA3_1-405B - tags: get,ml-model,llama3 diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index 7d5dd3fbf..8b79d5931 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -1018,6 +1018,12 @@ variations: names: - igbh-original - igbh-dataset + - tags: get,ml-model,rgat,_r2-downloader,_mlcommons + enable_if_env: + MLC_USE_ML_MODEL_FROM_HOST: + - 'yes' + names: + - rgat-model pointpainting: group: diff --git a/script/get-ml-model-rgat/meta.yaml b/script/get-ml-model-rgat/meta.yaml index b9ffe25c1..4cc77206c 100644 --- a/script/get-ml-model-rgat/meta.yaml +++ b/script/get-ml-model-rgat/meta.yaml @@ -15,8 +15,6 @@ new_env_keys: - RGAT_CHECKPOINT_PATH prehook_deps: - enable_if_env: - MLC_DOWNLOAD_TOOL: - - rclone MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' env: @@ -24,7 +22,7 @@ prehook_deps: extra_cache_tags: rgat,gnn,model,ml-model force_cache: true names: - - download-file + - download-rgat-model-file tags: download,file force_env_keys: - MLC_OUTDIRNAME @@ -52,18 +50,30 @@ variations: mlcommons: default: true default_variations: - download-tool: rclone + download-tool: r2-downloader group: download-source rclone: adr: - download-file: + download-rgat-model-file: tags: _rclone env: MLC_DOWNLOAD_TOOL: rclone MLC_RCLONE_CONFIG_NAME: mlc-inference group: download-tool - rclone,fp32: + r2-downloader: + adr: + download-rgat-model-file: + tags: _r2-downloader + env: + MLC_DOWNLOAD_TOOL: r2-downloader + group: download-tool + rclone,mlcommons,fp32: env: MLC_ML_MODEL_STARTING_WEIGHTS_FILENAME: https://github.com/mlcommons/inference/tree/master/graph/R-GAT#download-model-using-rclone MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/R-GAT/RGAT.pt MLC_DOWNLOAD_FILENAME: RGAT + r2-downloader,mlcommons,fp32: + env: + MLC_ML_MODEL_STARTING_WEIGHTS_FILENAME: https://github.com/mlcommons/inference/tree/master/graph/R-GAT#download-model-using-mlc-r2-downloader + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/rgat-model.uri + MLC_DOWNLOAD_FILENAME: RGAT From 661c633a86a6fc3c94e01cf64dd6df2ae23ae1a3 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Sat, 4 Oct 2025 11:26:42 +0530 Subject: [PATCH 03/55] Add tests and dry run option --- script/get-ml-model-rgat/meta.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/script/get-ml-model-rgat/meta.yaml b/script/get-ml-model-rgat/meta.yaml index 4cc77206c..2ab088b9f 100644 --- a/script/get-ml-model-rgat/meta.yaml +++ b/script/get-ml-model-rgat/meta.yaml @@ -77,3 +77,18 @@ variations: MLC_ML_MODEL_STARTING_WEIGHTS_FILENAME: https://github.com/mlcommons/inference/tree/master/graph/R-GAT#download-model-using-mlc-r2-downloader MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/rgat-model.uri MLC_DOWNLOAD_FILENAME: RGAT + dry-run: + env: + MLC_DOWNLOAD_MODE: dry + group: run-mode + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x +tests: + run_inputs: + - variations_list: + - rclone,mlcommons,fp32,dry-run + - r2-downloader,mlcommons,fp32,dry-run From fc8cf79497c40c09c92142b738315cff48a20411 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Sat, 4 Oct 2025 12:46:30 +0530 Subject: [PATCH 04/55] Support r2-downloader for llama3.1-405b asset downloads --- .../app-mlperf-inference-mlcommons-python/meta.yaml | 2 +- script/app-mlperf-inference/meta.yaml | 7 +++++++ script/get-dataset-mlperf-inference-llama3/meta.yaml | 11 ++++------- script/get-ml-model-llama3/meta.yaml | 2 +- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index 3fdff095d..3077a1835 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -701,7 +701,7 @@ deps: - waymo-dataset ## llama3_1 dataset - - tags: get,dataset,mlperf,inference,llama3,_validation + - tags: get,dataset,mlperf,inference,llama3,_validation,_r2-downloader names: - llama3_1-dataset - llama3-dataset diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index 7d5dd3fbf..7875ae857 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -1225,6 +1225,13 @@ variations: names: - llama3_1-405b - llama3-405b + - tags: mlcr get,dataset,mlperf,inference,llama3,_validation,_r2-downloader + enable_if_env: + MLC_USE_DATASET_FROM_HOST: + - 'yes' + names: + - llama3_1-dataset + - llama3-dataset deepseek-r1: group: diff --git a/script/get-dataset-mlperf-inference-llama3/meta.yaml b/script/get-dataset-mlperf-inference-llama3/meta.yaml index 24aef9e20..929f03732 100644 --- a/script/get-dataset-mlperf-inference-llama3/meta.yaml +++ b/script/get-dataset-mlperf-inference-llama3/meta.yaml @@ -33,7 +33,6 @@ variations: default: true group: dataset-type env: - MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl MLC_DATASET_TYPE: validation MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl validation,rclone: @@ -42,12 +41,11 @@ variations: MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl validation,r2-downloader: env: - MLC_RCLONE_URL: "" - MLC_DATASET_FILE_NAME: "" + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/llama3-1-405b-dataset-8313.uri + MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl calibration: group: dataset-type env: - MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl MLC_DATASET_TYPE: calibration MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl calibration,rclone: @@ -56,8 +54,8 @@ variations: MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl calibration,r2-downloader: env: - MLC_RCLONE_URL: "" - MLC_DATASET_FILE_NAME: "" + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/llama3-1-405b-calibration-dataset-512.uri + MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl rclone: add_deps_recursive: dae: @@ -74,7 +72,6 @@ variations: tags: _r2-downloader env: MLC_DOWNLOAD_FILENAME: checkpoint - MLC_RCLONE_CONFIG_NAME: mlc-inference group: download-tool print_env_at_the_end: MLC_DATASET_LLAMA3_PATH: Path to the dataset diff --git a/script/get-ml-model-llama3/meta.yaml b/script/get-ml-model-llama3/meta.yaml index b10665d20..d316d1f45 100644 --- a/script/get-ml-model-llama3/meta.yaml +++ b/script/get-ml-model-llama3/meta.yaml @@ -74,8 +74,8 @@ variations: enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - yes - default: true r2-downloader: + default: true group: download-tool add_deps_recursive: dae: From b4dc54139b595cb72058113500eed9cdae234afc Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 5 Oct 2025 05:43:24 +0530 Subject: [PATCH 05/55] Fix #659, hpcx_paths usage in nvidia implementation --- script/build-mlperf-inference-server-nvidia/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/build-mlperf-inference-server-nvidia/customize.py b/script/build-mlperf-inference-server-nvidia/customize.py index f6a2f399f..3966021ac 100644 --- a/script/build-mlperf-inference-server-nvidia/customize.py +++ b/script/build-mlperf-inference-server-nvidia/customize.py @@ -19,8 +19,8 @@ def preprocess(i): env['+LIBRARY_PATH'].append(os.path.join( env['MLC_TENSORRT_INSTALL_PATH'], "lib")) + hpcx_paths = [] if is_true(env.get('BUILD_TRTLLM')): - hpcx_paths = [] if os.path.exists("/opt/hpcx/ucx/lib"): hpcx_paths.append("/opt/hpcx/ucx/lib") if os.path.exists("/opt/hpcx/ucc/lib"): From 7306680c6e10b67b09104649f16b7f20593ae112 Mon Sep 17 00:00:00 2001 From: Arjun Date: Mon, 6 Oct 2025 18:58:14 +0530 Subject: [PATCH 06/55] Support openmp in llvm build --- script/install-llvm-src/customize.py | 8 +++++++- script/install-llvm-src/meta.yaml | 23 +++++++++++++++++++++-- script/install-llvm-src/run.sh | 2 +- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/script/install-llvm-src/customize.py b/script/install-llvm-src/customize.py index 31a914c85..8c78ec22e 100644 --- a/script/install-llvm-src/customize.py +++ b/script/install-llvm-src/customize.py @@ -16,7 +16,13 @@ def preprocess(i): q = '"' if os_info['platform'] == 'windows' else "'" clang_file_name = "clang" - extra_cmake_options = env.get('MLC_LLVM_EXTRA_CMAKE_OPTIONS', '') + + if env.get('+MLC_LLVM_CMAKE_OPTIONS', '') != '': + cmake_options = " ".join(env['+MLC_LLVM_CMAKE_OPTIONS']) + else: + cmake_options = '' + + extra_cmake_options = cmake_options + env.get('MLC_LLVM_EXTRA_CMAKE_OPTIONS', '') if env.get('MLC_LLVM_INSTALLED_PATH', '') != '' and os.path.exists( env.get('MLC_LLVM_INSTALLED_PATH')): diff --git a/script/install-llvm-src/meta.yaml b/script/install-llvm-src/meta.yaml index d145e44c6..668139919 100644 --- a/script/install-llvm-src/meta.yaml +++ b/script/install-llvm-src/meta.yaml @@ -86,6 +86,8 @@ variations: +MLC_LLVM_ENABLE_PROJECTS: - clang group: clang + no-clang: + group: clang flang: env: +MLC_LLVM_ENABLE_PROJECTS: @@ -116,11 +118,19 @@ variations: env: +MLC_LLVM_ENABLE_PROJECTS: - lld + +MLC_LLVM_CMAKE_OPTIONS: + - DCLANG_DEFAULT_LINKER=lld group: lld - no-clang: - group: clang no-lld: group: lld + openmp: + default: true + env: + +MLC_LLVM_ENABLE_PROJECTS: + - openmp + group: openmp + no-openmp: + group: openmp debug: env: MLC_LLVM_BUILD_TYPE: debug @@ -276,6 +286,15 @@ variations: - full-history env: MLC_GIT_CHECKOUT_TAG: '#' + no-tests: + group: tests + default: true + env: + MLC_LLVM_CHECK_ALL: '' + tests: + group: tests + env: + MLC_LLVM_CHECK_ALL: check-all docker: skip_run_cmd: true pre_run_cmds: diff --git a/script/install-llvm-src/run.sh b/script/install-llvm-src/run.sh index d71086ce1..2f33db19d 100644 --- a/script/install-llvm-src/run.sh +++ b/script/install-llvm-src/run.sh @@ -28,7 +28,7 @@ if [ ! -d "${INSTALL_DIR}" ] || [ ${MLC_LLVM_CONDA_ENV} == "yes" ]; then echo "${MLC_LLVM_CMAKE_CMD}" eval "${MLC_LLVM_CMAKE_CMD}" - ninja + ninja ${MLC_LLVM_CHECK_ALL} if [ "${?}" != "0" ]; then exit 1; fi ninja install if [ "${?}" != "0" ]; then exit 1; fi From df01dbee75f90a3d9753c503a6491aea76831cd2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 6 Oct 2025 13:28:33 +0000 Subject: [PATCH 07/55] [Automated Commit] Format Codebase [skip ci] --- script/install-llvm-src/customize.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/script/install-llvm-src/customize.py b/script/install-llvm-src/customize.py index 8c78ec22e..12680c47e 100644 --- a/script/install-llvm-src/customize.py +++ b/script/install-llvm-src/customize.py @@ -16,13 +16,14 @@ def preprocess(i): q = '"' if os_info['platform'] == 'windows' else "'" clang_file_name = "clang" - + if env.get('+MLC_LLVM_CMAKE_OPTIONS', '') != '': cmake_options = " ".join(env['+MLC_LLVM_CMAKE_OPTIONS']) else: cmake_options = '' - - extra_cmake_options = cmake_options + env.get('MLC_LLVM_EXTRA_CMAKE_OPTIONS', '') + + extra_cmake_options = cmake_options + \ + env.get('MLC_LLVM_EXTRA_CMAKE_OPTIONS', '') if env.get('MLC_LLVM_INSTALLED_PATH', '') != '' and os.path.exists( env.get('MLC_LLVM_INSTALLED_PATH')): From e8586c860ca37ba5fca2733c52b63c127044cdef Mon Sep 17 00:00:00 2001 From: Arjun Date: Mon, 6 Oct 2025 19:02:17 +0530 Subject: [PATCH 08/55] Support openmp in llvm build --- script/install-llvm-src/run.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/script/install-llvm-src/run.sh b/script/install-llvm-src/run.sh index 2f33db19d..52178a27a 100644 --- a/script/install-llvm-src/run.sh +++ b/script/install-llvm-src/run.sh @@ -24,14 +24,20 @@ if [ ! -d "${INSTALL_DIR}" ] || [ ${MLC_LLVM_CONDA_ENV} == "yes" ]; then echo "******************************************************" cd build - if [ "${?}" != "0" ]; then exit 1; fi + test $? -eq 0 || exit $? echo "${MLC_LLVM_CMAKE_CMD}" eval "${MLC_LLVM_CMAKE_CMD}" - ninja ${MLC_LLVM_CHECK_ALL} - if [ "${?}" != "0" ]; then exit 1; fi - ninja install - if [ "${?}" != "0" ]; then exit 1; fi + + cmd="ninja ${MLC_LLVM_CHECK_ALL}" + echo $cmd + eval $cmd + test $? -eq 0 || exit $? + + cmd="ninja install" + echo $cmd + eval $cmd + test $? -eq 0 || exit $? fi From 30615e6ee36c88036dd8d37757dcc90e4a7a5c8b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 6 Oct 2025 19:55:45 +0530 Subject: [PATCH 09/55] Added cache_expiration for get-git-repo --- script/get-git-repo/meta.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/script/get-git-repo/meta.yaml b/script/get-git-repo/meta.yaml index e52110035..2bd9e61da 100644 --- a/script/get-git-repo/meta.yaml +++ b/script/get-git-repo/meta.yaml @@ -2,13 +2,14 @@ alias: get-git-repo automation_alias: script automation_uid: 5b4e0237da074764 cache: true +cache_expiration: 5d category: DevOps automation default_env: MLC_GIT_CHECKOUT_FOLDER: repo MLC_GIT_DEPTH: --depth 4 MLC_GIT_PATCH: 'no' MLC_GIT_RECURSE_SUBMODULES: ' --recurse-submodules' - MLC_GIT_URL: https://github.com/mlcommons/ck.git + MLC_GIT_URL: https://github.com/mlcommons/mlperf-automations deps: - tags: detect,os input_mapping: @@ -80,6 +81,7 @@ variations: MLC_GIT_URL: '#' group: repo sha.#: + cache_expiration: 500d default_variations: git-history: full-history env: @@ -94,6 +96,7 @@ variations: env: MLC_GIT_SUBMODULES: '#' tag.#: + cache_expiration: 500d env: MLC_GIT_CHECKOUT_TAG: '#' group: checkout From c897096cc5acc8570d8bcb25cbe370128769a703 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 6 Oct 2025 19:58:37 +0530 Subject: [PATCH 10/55] Added cache_expiration for install-llvm-src --- script/install-llvm-src/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/install-llvm-src/meta.yaml b/script/install-llvm-src/meta.yaml index 668139919..2f1c906da 100644 --- a/script/install-llvm-src/meta.yaml +++ b/script/install-llvm-src/meta.yaml @@ -78,6 +78,7 @@ variations: env: MLC_GIT_CHECKOUT_TAG: '#' branch.#: + cache_expiration: 5d env: MLC_GIT_CHECKOUT: '#' clang: From 529e658881d549b7511f0f635d6098561a8a53f8 Mon Sep 17 00:00:00 2001 From: Arjun Date: Mon, 6 Oct 2025 20:20:59 +0530 Subject: [PATCH 11/55] Support cache_expiration from input --- automation/script/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/script/module.py b/automation/script/module.py index 399196145..d633628f4 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -799,7 +799,7 @@ def _run(self, i): run_state['script_repo_git'] = script_item.repo.meta.get( 'git', False) run_state['cache'] = meta.get('cache', False) - run_state['cache_expiration'] = meta.get('cache_expiration', False) + run_state['cache_expiration'] = i.get('cache_expiration', meta.get('cache_expiration', False)) if not recursion: run_state['script_entry_repo_to_report_errors'] = meta.get( From 8de09c9140b4dca524e276071f6be460473bd20f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 6 Oct 2025 14:51:30 +0000 Subject: [PATCH 12/55] [Automated Commit] Format Codebase [skip ci] --- automation/script/module.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/automation/script/module.py b/automation/script/module.py index d633628f4..9be00489e 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -799,7 +799,9 @@ def _run(self, i): run_state['script_repo_git'] = script_item.repo.meta.get( 'git', False) run_state['cache'] = meta.get('cache', False) - run_state['cache_expiration'] = i.get('cache_expiration', meta.get('cache_expiration', False)) + run_state['cache_expiration'] = i.get( + 'cache_expiration', meta.get( + 'cache_expiration', False)) if not recursion: run_state['script_entry_repo_to_report_errors'] = meta.get( From 82652eb747de27583352bb638f997991ec7b0bc0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 6 Oct 2025 15:20:54 +0000 Subject: [PATCH 13/55] [Automated Commit] Document script/get-ml-model-rgat/meta.yaml [skip ci] --- script/get-ml-model-rgat/README.md | 83 ++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 script/get-ml-model-rgat/README.md diff --git a/script/get-ml-model-rgat/README.md b/script/get-ml-model-rgat/README.md new file mode 100644 index 000000000..957bb745f --- /dev/null +++ b/script/get-ml-model-rgat/README.md @@ -0,0 +1,83 @@ +# README for get-ml-model-rgat +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,raw,ml-model,rgat +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--checkpoint` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Download-source + +- `mlcommons` (default) + +### Download-tool + +- `r2-downloader` +- `rclone` + +### Precision + +- `fp32` (default) + +### Run-mode + +- `dry-run` + +### Ungrouped + +- `pytorch` From 136f5c43d8b4ed6cb2cc5132a2a8f405d169e621 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 6 Oct 2025 15:26:32 +0000 Subject: [PATCH 14/55] [Automated Commit] Document script/install-llvm-src/meta.yaml [skip ci] --- script/install-llvm-src/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/script/install-llvm-src/README.md b/script/install-llvm-src/README.md index 7a596672b..baffb8c4f 100644 --- a/script/install-llvm-src/README.md +++ b/script/install-llvm-src/README.md @@ -92,10 +92,20 @@ mlcr install,src,llvm,from.src,src-llvm - `lld` (default) - `no-lld` +### Openmp + +- `no-openmp` +- `openmp` (default) + ### Repo - `repo.#` _(# can be substituted dynamically)_ +### Tests + +- `no-tests` (default) +- `tests` + ### Ungrouped - `branch.#` _(# can be substituted dynamically)_ From dcdf5c09209d7ad1c99be59fa4d94d3cf045c821 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 6 Oct 2025 21:15:30 +0530 Subject: [PATCH 15/55] Support docker privileged as input argument (#661) * Support docker privileged as input argument --- automation/script/docker_utils.py | 2 +- script/app-mlperf-inference/meta.yaml | 4 ++-- script/install-llvm-src/meta.yaml | 2 +- script/run-docker-container/meta.yaml | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index d2f0488e6..79af8f41e 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -145,7 +145,7 @@ def prepare_docker_inputs(input_params, docker_settings, keys += [ "skip_run_cmd", "pre_run_cmds", "run_cmd_prefix", "all_gpus", "num_gpus", "device", "gh_token", "port_maps", "shm_size", "pass_user_id", "pass_user_group", "extra_run_args", "detached", "interactive", - "dt", "it", "use_host_group_id", "use_host_user_id", "keep_detached", "reuse_existing", "use_google_dns" + "dt", "it", "use_host_group_id", "use_host_user_id", "keep_detached", "reuse_existing", "use_google_dns", "privileged" ] # Collect Dockerfile inputs docker_inputs = { diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index 9a12ed652..56fc83128 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -704,7 +704,7 @@ variations: implementation docker: interactive: True - extra_run_args: ' --privileged' + privileged: True mounts: - "${{ MLC_MLPERF_INFERENCE_INTEL_GPTJ_INT8_MODEL_PATH }}:${{ MLC_MLPERF_INFERENCE_INTEL_GPTJ_INT8_MODEL_PATH }}" - "${{ GPTJ_CHECKPOINT_PATH }}:${{ GPTJ_CHECKPOINT_PATH }}" @@ -748,7 +748,7 @@ variations: implementation docker: interactive: True - extra_run_args: ' --privileged' + privileged: True mounts: - "${{ LLAMA2_CHECKPOINT_PATH }}:${{ LLAMA2_CHECKPOINT_PATH }}" - "${{ GPTJ_CHECKPOINT_PATH }}:${{ GPTJ_CHECKPOINT_PATH }}" diff --git a/script/install-llvm-src/meta.yaml b/script/install-llvm-src/meta.yaml index 2f1c906da..42189a7d4 100644 --- a/script/install-llvm-src/meta.yaml +++ b/script/install-llvm-src/meta.yaml @@ -120,7 +120,7 @@ variations: +MLC_LLVM_ENABLE_PROJECTS: - lld +MLC_LLVM_CMAKE_OPTIONS: - - DCLANG_DEFAULT_LINKER=lld + - '-DCLANG_DEFAULT_LINKER=lld' group: lld no-lld: group: lld diff --git a/script/run-docker-container/meta.yaml b/script/run-docker-container/meta.yaml index 4cb3b1ed6..47c6607e2 100644 --- a/script/run-docker-container/meta.yaml +++ b/script/run-docker-container/meta.yaml @@ -50,6 +50,7 @@ input_mapping: port_maps: MLC_DOCKER_PORT_MAPS post_run_cmds: MLC_DOCKER_POST_RUN_COMMANDS pre_run_cmds: MLC_DOCKER_PRE_RUN_COMMANDS + privileged: MLC_DOCKER_PRIVILEGED_MODE real_run: MLC_REAL_RUN recreate: MLC_DOCKER_IMAGE_RECREATE rebuild: MLC_DOCKER_IMAGE_RECREATE From 6377da53433646255dd7e01b6b0cb0e41bed2e0d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 6 Oct 2025 15:45:57 +0000 Subject: [PATCH 16/55] [Automated Commit] Document script/run-docker-container/meta.yaml [skip ci] --- script/run-docker-container/README.md | 102 ++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 script/run-docker-container/README.md diff --git a/script/run-docker-container/README.md b/script/run-docker-container/README.md new file mode 100644 index 000000000..7897f88ff --- /dev/null +++ b/script/run-docker-container/README.md @@ -0,0 +1,102 @@ +# README for run-docker-container +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr run,docker,container +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--all_gpus` | | | `` | +| `--num_gpus` | | | `` | +| `--base` | | | `` | +| `--cache` | | | `` | +| `--mlc_repo` | | | `` | +| `--detached` | | | `` | +| `--device` | | | `` | +| `--docker_image_base` | Alias for base | | `` | +| `--docker_base_image` | Alias for base | | `` | +| `--base_image` | Alias for base | | `` | +| `--keep_detached` | | | `` | +| `--reuse_existing` | | | `no` | +| `--docker_os` | | | `` | +| `--docker_os_version` | | | `` | +| `--os` | Alias for docker_os | | `` | +| `--os_version` | Alias for docker_os_version | | `` | +| `--extra_run_args` | | | `` | +| `--fake_run_option` | | | `` | +| `--gh_token` | | | `` | +| `--image_name` | | | `` | +| `--image_repo` | | | `` | +| `--image_tag` | | | `` | +| `--image_tag_extra` | | | `` | +| `--interactive` | | | `` | +| `--it` | | | `` | +| `--mounts` | | | `` | +| `--pass_user_id` | | | `` | +| `--pass_user_group` | | | `` | +| `--port_maps` | | | `` | +| `--post_run_cmds` | | | `` | +| `--pre_run_cmds` | | | `` | +| `--privileged` | | | `no` | +| `--real_run` | | | `` | +| `--recreate` | | | `` | +| `--rebuild` | Alias for recreate | | `` | +| `--run_cmd` | | | `` | +| `--run_cmd_extra` | | | `` | +| `--save_script` | | | `` | +| `--script_tags` | | | `` | +| `--shm_size` | | | `` | +| `--use_google_dns` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations From 2b33ec891cdab0a629d596d60f7e8655e408980f Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:16:28 +0530 Subject: [PATCH 17/55] Support r2-downloader for nuscenes dataset download (#667) * Initial changes for supporting r2-downloader for nuscenes dataset * code formatting and update for r2-downloader --- .../customize.py | 34 +++-- .../meta.yaml | 119 ++++++++++++------ .../get-preprocessed-dataset-nuscenes/run.sh | 14 ++- 3 files changed, 112 insertions(+), 55 deletions(-) diff --git a/script/get-preprocessed-dataset-nuscenes/customize.py b/script/get-preprocessed-dataset-nuscenes/customize.py index 5236b5c78..635994ec0 100644 --- a/script/get-preprocessed-dataset-nuscenes/customize.py +++ b/script/get-preprocessed-dataset-nuscenes/customize.py @@ -19,18 +19,26 @@ def postprocess(i): env = i['env'] if is_true(env.get('MLC_TMP_REQUIRE_DOWNLOAD', '')): - env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'] = os.path.join( - env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'], - env['MLC_DATASET_NUSCENES_EXTRACTED_FOLDER_NAME']) - if env.get( - 'MLC_PREPROCESSED_DATASET_NUSCENES_SCENE_LENGTHS_PATH', '') != '': - shutil.copy( - os.path.join( - env['MLC_PREPROCESSED_DATASET_NUSCENES_SCENE_LENGTHS_PATH'], - env['MLC_DATASET_NUSCENES_SCENE_PICKLE_FILENAME']), - os.path.join( - os.path.dirname( - env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'].rstrip("/")), - env['MLC_DATASET_NUSCENES_SCENE_PICKLE_FILENAME'])) + if env.get('MLC_DOWNLOAD_TOOL', '') == "rclone": + env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'] = os.path.join( + env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'], + env['MLC_DATASET_NUSCENES_EXTRACTED_FOLDER_NAME']) + elif env.get('MLC_DOWNLOAD_TOOL', '') == "r2-downloader": + env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'] = os.path.join( + env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'], + "preprocessed", + env['MLC_DATASET_NUSCENES_EXTRACTED_FOLDER_NAME']) + + if env.get('MLC_DOWNLOAD_TOOL', '') == "rclone": + if env.get( + 'MLC_PREPROCESSED_DATASET_NUSCENES_SCENE_LENGTHS_PATH', '') != '': + shutil.copy( + os.path.join( + env['MLC_PREPROCESSED_DATASET_NUSCENES_SCENE_LENGTHS_PATH'], + env['MLC_DATASET_NUSCENES_SCENE_PICKLE_FILENAME']), + os.path.join( + os.path.dirname( + env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'].rstrip("/")), + env['MLC_DATASET_NUSCENES_SCENE_PICKLE_FILENAME'])) return {'return': 0} diff --git a/script/get-preprocessed-dataset-nuscenes/meta.yaml b/script/get-preprocessed-dataset-nuscenes/meta.yaml index 6436574a6..b53494c4a 100644 --- a/script/get-preprocessed-dataset-nuscenes/meta.yaml +++ b/script/get-preprocessed-dataset-nuscenes/meta.yaml @@ -17,42 +17,59 @@ print_env_at_the_end: MLC_PREPROCESSED_DATASET_NUSCENES_PATH: Preprocessed Nuscenes dataset path MLC_PREPROCESSED_DATASET_NUSCENES_ACC_CHECKER_MIN_FILES_PATH: Path containing minimum files for accuracy checker variations: + rclone: + group: download-tool + env: + MLC_DOWNLOAD_TOOL: rclone + add_deps_recursive: + dae: + tags: _rclone + dae_sl: + tags: _rclone + dae_ac: + tags: _rclone + default: true + r2-downloader: + env: + MLC_DOWNLOAD_TOOL: r2-downloader + add_deps_recursive: + dae: + tags: _r2-downloader + dae_sl: + tags: _r2-downloader + dae_ac: + tags: _r2-downloader + group: download-tool + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + prebuilt: + default: true + group: dataset-src + env: + MLC_NUSCENES_DATASET_TYPE: prebuilt validation: default: true group: dataset-type env: MLC_DATASET_NUSCENES_EXTRACTED_FOLDER_NAME: val_3d MLC_DATASET_NUSCENES_TAR_FILENAME: val_3d.tar.gz - MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset/preprocessed/<<>> calibration: group: dataset-type env: MLC_DATASET_NUSCENES_EXTRACTED_FOLDER_NAME: calib_3d MLC_DATASET_NUSCENES_TAR_FILENAME: calib_3d.tar.gz - MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset/preprocessed/<<>> - prebuilt: - default: true - group: dataset-src - env: - MLC_NUSCENES_DATASET_TYPE: prebuilt - MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_PATH - MLC_EXTRACT_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_PATH - MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' mlc: group: download-src default: true prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_config-name.mlc-nuscenes - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - env: - MLC_RCLONE_DRIVE_FOLDER_ID: 17CpM5eU8tjrxh_LpH_BTNTeT37PhzcnC - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' @@ -73,7 +90,30 @@ variations: MLC_DATASET_NUSCENES_SCENE_PICKLE_FILENAME: scene_lengths.pkl MLC_DATASET_NUSCENES_ACC_CHECKER_DEP_FILES_TAR_NAME: nuscenes_min.tar.gz MLC_DATASET_NUSCENES_ACC_REQ_FILE_EXTRACTED_FOLDER_NAME: nuscenes + prebuilt,mlc,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' + prebuilt,mlc: + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_PATH + validation,mlc,rclone: prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + MLC_DOWNLOAD_TOOL: + - rclone + - tags: get,rclone-config,_config-name.mlc-nuscenes + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + MLC_DOWNLOAD_TOOL: + - rclone + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 17CpM5eU8tjrxh_LpH_BTNTeT37PhzcnC - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' @@ -87,10 +127,10 @@ variations: env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_SCENE_LENGTHS_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_SCENE_LENGTHS_PATH - MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset/scene_lengths.pkl + MLC_DOWNLOAD_NUSCENES_SCENE_LENGTHS_URL: mlc-nuscenes:nuscenes_dataset/scene_lengths.pkl update_tags_from_env_with_prefix: _url.: - - MLC_DOWNLOAD_URL + - MLC_DOWNLOAD_NUSCENES_SCENE_LENGTHS_URL - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' @@ -104,29 +144,26 @@ variations: env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_ACC_CHECKER_MIN_FILES_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_ACC_CHECKER_MIN_FILES_PATH - MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset/nuscenes_min.tar.gz + MLC_DOWNLOAD_ACC_CHECKER_MIN_FILES_URL: mlc-nuscenes:nuscenes_dataset/nuscenes_min.tar.gz update_tags_from_env_with_prefix: _url.: - - MLC_DOWNLOAD_URL - rclone: - group: download-tool - add_deps_recursive: - dae: - tags: _rclone - dae_sl: - tags: _rclone - dae_ac: - tags: _rclone - default: true - dry-run: - group: run-mode + - MLC_DOWNLOAD_NUSCENES_SCENE_LENGTHS_URL env: - MLC_DOWNLOAD_MODE: dry - dry-run,rclone: + MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset/preprocessed/<<>> + calibration,mlc,rclone: + MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset/preprocessed/<<>> + validation,mlc,r2-downloader: env: - MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + MLC_DOWNLOAD_URL: https://nuscenes.mlcommons-storage.org/metadata/nuscenes-dataset.uri + calibration,mlc,r2-downloader: + MLC_DOWNLOAD_URL: https://nuscenes.mlcommons-storage.org/metadata/nuscenes-dataset.uri tests: + needs_pat: true run_inputs: - variations_list: - validation,prebuilt,rclone,mlc,dry-run - - calibration,prebuilt,rclone,mlc,dry-run \ No newline at end of file + - calibration,prebuilt,rclone,mlc,dry-run + - validation,prebuilt,r2-downloader,mlc,dry-run + + - calibration,prebuilt,r2-downloader,mlc,dry-run + diff --git a/script/get-preprocessed-dataset-nuscenes/run.sh b/script/get-preprocessed-dataset-nuscenes/run.sh index 16337c44d..1954ef736 100644 --- a/script/get-preprocessed-dataset-nuscenes/run.sh +++ b/script/get-preprocessed-dataset-nuscenes/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" ]]; then +if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" && "$MLC_DOWNLOAD_TOOL" = "rclone" ]]; then cd "${MLC_PREPROCESSED_DATASET_NUSCENES_PATH}" || exit for f in *.tar.gz; do tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } @@ -10,4 +10,16 @@ if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" ]]; t tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } done cd - || exit +fi + +if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" && "$MLC_DOWNLOAD_TOOL" = "r2-downloader" ]]; then + cd "${MLC_PREPROCESSED_DATASET_NUSCENES_PATH}/preprocessed" || exit + for f in *.tar.gz; do + tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + done + cd "${MLC_PREPROCESSED_DATASET_NUSCENES_PATH}" || exit + for f in *.tar.gz; do + tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + done + cd - || exit fi \ No newline at end of file From 215b2ddfed4db0201381f8b3bd58154def7e6759 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 8 Oct 2025 06:46:57 +0000 Subject: [PATCH 18/55] [Automated Commit] Document script/get-preprocessed-dataset-nuscenes/meta.yaml [skip ci] --- .../README.md | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 script/get-preprocessed-dataset-nuscenes/README.md diff --git a/script/get-preprocessed-dataset-nuscenes/README.md b/script/get-preprocessed-dataset-nuscenes/README.md new file mode 100644 index 000000000..5b51c8db4 --- /dev/null +++ b/script/get-preprocessed-dataset-nuscenes/README.md @@ -0,0 +1,80 @@ +# README for get-preprocessed-dataset-nuscenes +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,dataset,nuscenes,preprocessed +``` + +No script specific inputs +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Dataset-src + +- `prebuilt` (default) + +### Dataset-type + +- `calibration` +- `validation` (default) + +### Download-src + +- `mlc` (default) + +### Download-tool + +- `r2-downloader` +- `rclone` (default) + +### Run-mode + +- `dry-run` From 0f75b00bd2ded7c18b5006fa0ca00ccac3df18f6 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:17:44 +0530 Subject: [PATCH 19/55] Changes to support the new compliance directory structure (#666) * Changes to support the new compliance directory structure * Enable backward compatability --- script/app-mlperf-inference/customize.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/script/app-mlperf-inference/customize.py b/script/app-mlperf-inference/customize.py index 62c05a287..fb346e496 100644 --- a/script/app-mlperf-inference/customize.py +++ b/script/app-mlperf-inference/customize.py @@ -504,12 +504,21 @@ def postprocess(i): COMPLIANCE_DIR = output_dir OUTPUT_DIR = os.path.dirname(COMPLIANCE_DIR) + # For backward compatibility with inference commit + # d9efabf829526234629a4dfe5cc34a36cb81417c SCRIPT_PATH = os.path.join( env['MLC_MLPERF_INFERENCE_SOURCE'], "compliance", "nvidia", test, "run_verification.py") + if not os.path.exists(SCRIPT_PATH): + SCRIPT_PATH = os.path.join( + env['MLC_MLPERF_INFERENCE_SOURCE'], + "compliance", + test, + "run_verification.py") + if test == "TEST06": cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {q}{SCRIPT_PATH}{q} -c {q}{COMPLIANCE_DIR}{q} -o {q}{OUTPUT_DIR}{q} --scenario {scenario} --dtype int32""" else: @@ -523,8 +532,14 @@ def postprocess(i): run_script_input = i['run_script_input'] automation = i['automation'] + # For backward compatibility with inference commit + # d9efabf829526234629a4dfe5cc34a36cb81417c SCRIPT_PATH = os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "compliance", "nvidia", test, "create_accuracy_baseline.sh") + if not os.path.exists(SCRIPT_PATH): + SCRIPT_PATH = os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "compliance", test, + "create_accuracy_baseline.sh") + TEST01_DIR = os.path.join(OUTPUT_DIR, "TEST01") OUTPUT_DIR = os.path.join(OUTPUT_DIR, "TEST01", "accuracy") if not os.path.exists(OUTPUT_DIR): From 7632625b0e50065f69cfafc99cab6f3414a23a4d Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 9 Oct 2025 19:59:24 +0530 Subject: [PATCH 20/55] Support ssd model download through r2-downloader (#669) --- .../get-ml-model-abtf-ssd-pytorch/meta.yaml | 67 ++++++++++++++----- 1 file changed, 50 insertions(+), 17 deletions(-) diff --git a/script/get-ml-model-abtf-ssd-pytorch/meta.yaml b/script/get-ml-model-abtf-ssd-pytorch/meta.yaml index fa1b5d39a..42f39b2f3 100644 --- a/script/get-ml-model-abtf-ssd-pytorch/meta.yaml +++ b/script/get-ml-model-abtf-ssd-pytorch/meta.yaml @@ -160,6 +160,20 @@ variations: rclone: group: download-tool + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config + force_cache: true + names: + - rclone-config + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + MLC_DOWNLOAD_SRC: + - mlcommons env: MLC_RCLONE_COPY_USING: copyurl adr: @@ -186,29 +200,55 @@ variations: env: MLC_MODEL_FORMAT: onnx - onnx,mlc: + onnx,mlc,rclone: env: MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_ssd/ssd_resnet50.onnx MLC_ML_MODEL_FILENAME: ssd_resnet50.onnx + MLC_DOWNLOAD_URL: 'mlc-cognata:mlc_cognata_dataset/<<>>' + MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' + + onnx,mlc,r2-downloader: + env: + MLC_ML_MODEL_FILENAME: ssd_resnet50.onnx + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/ssd_checkpoint_onnx.uri pytorch: group: model-format env: MLC_MODEL_FORMAT: pth - pytorch,mlc: + pytorch,mlc,rclone: env: MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_ssd/baseline_8MP_ss_scales_fm1_5x5_all_ep60.pth MLC_ML_MODEL_FILENAME: baseline_8MP_ss_scales_fm1_5x5_all_ep60.pth + MLC_DOWNLOAD_URL: 'mlc-cognata:mlc_cognata_dataset/<<>>' + MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' + + pytorch,mlc,r2-downloader: + env: + MLC_ML_MODEL_FILENAME: baseline_8MP_ss_scales_fm1_5x5_all_ep60.pth + MLC_DOWNLOAD_URL: 'https://cognata.mlcommons-storage.org/metadata/ssd_checkpoint_pth.uri' rclone,mlc: group: download-tool add_deps_recursive: - dae: + dae-ssd: tags: _rclone + rclone-config: + tags: _config-name.mlc-cognata + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 env: MLC_RCLONE_COPY_USING: sync + r2-downloader: + group: download-tool + + r2-downloader,mlc: + adr: + dae-ssd: + tags: _r2-downloader + dry-run: group: run-mode env: @@ -218,32 +258,23 @@ variations: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + mlc: group: download-src prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_config-name.mlc-cognata - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - env: - MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_ML_MODEL_SSD_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_MLC_MODEL_SSD_PATH - MLC_DOWNLOAD_URL: 'mlc-cognata:mlc_cognata_dataset/<<>>' - MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' extra_cache_tags: ml,model,ssd,resnet50 force_cache: true names: - - dae + - dae-ssd tags: download-and-extract force_env_keys: - MLC_OUTDIRNAME @@ -258,3 +289,5 @@ tests: - variations_list: - onnx,rclone,mlc,dry-run - pytorch,rclone,mlc,dry-run + - onnx,r2-downloader,mlc,dry-run + - pytorch,r2-downloader,mlc,dry-run From f73dfaa8d3ed7c2499d92a06b5e0ca458fe64b1a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Oct 2025 14:29:57 +0000 Subject: [PATCH 21/55] [Automated Commit] Document script/get-ml-model-abtf-ssd-pytorch/meta.yaml [skip ci] --- script/get-ml-model-abtf-ssd-pytorch/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/script/get-ml-model-abtf-ssd-pytorch/README.md b/script/get-ml-model-abtf-ssd-pytorch/README.md index 6a6e5d631..7fc490756 100644 --- a/script/get-ml-model-abtf-ssd-pytorch/README.md +++ b/script/get-ml-model-abtf-ssd-pytorch/README.md @@ -69,6 +69,7 @@ mlcr get,ml-model,abtf-ssd-pytorch,ssd,resnet50,cmc ### Download-tool - `gdown` +- `r2-downloader` - `rclone` - `wget` (default) From 06b8e74390b7a2f2ce9cc17eebce5ef98eda99e7 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 9 Oct 2025 20:00:12 +0530 Subject: [PATCH 22/55] Support r2 downloader for deeplabv3+ (#670) --- script/get-ml-model-deeplabv3_plus/meta.yaml | 70 +++++++++++++++----- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/script/get-ml-model-deeplabv3_plus/meta.yaml b/script/get-ml-model-deeplabv3_plus/meta.yaml index 65c87e134..21184ac0b 100644 --- a/script/get-ml-model-deeplabv3_plus/meta.yaml +++ b/script/get-ml-model-deeplabv3_plus/meta.yaml @@ -20,48 +20,59 @@ variations: default: true env: MLC_MODEL_FORMAT: onnx - MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_deeplab/deeplabv3+_8mp.onnx MLC_ML_MODEL_FILENAME: deeplabv3+_8mp.onnx + onnx,mlc,rclone: + env: + MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_deeplab/deeplabv3+_8mp.onnx + onnx,mlc,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/deeplab_onnx.uri dynamic: {} onnx,dynamic: group: model-format env: MLC_MODEL_FORMAT: onnx - MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_deeplab/deeplabv3+_dynamic.onnx MLC_ML_MODEL_FILENAME: deeplabv3+_dynamic.onnx + onnx,dynamic,mlc,rclone: + env: + MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_deeplab/deeplabv3+_dynamic.onnx + onnx,dynamic,mlc,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/deeplab_dynamic_onnx.uri pytorch: group: model-format env: MLC_MODEL_FORMAT: pth - MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_deeplab/latest_deeplabv3plus_resnet50_cognata_os16_it100000.pth MLC_ML_MODEL_FILENAME: latest_deeplabv3plus_resnet50_cognata_os16_it100000.pth + pytorch,mlc,rclone: + env: + MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_deeplab/latest_deeplabv3plus_resnet50_cognata_os16_it100000.pth + pytorch,mlc,r2-downlaoder: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/deeplab_checkpoint.uri + rclone,mlc: + add_deps_recursive: + rclone-config: + tags: _config-name.mlc-cognata + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' + MLC_DOWNLOAD_URL: 'mlc-cognata:mlc_cognata_dataset/<<>>' mlc: group: download-src default: true prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_config-name.mlc-cognata - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - env: - MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_ML_MODEL_DEEPLABV3_PLUS_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_ML_MODEL_DEEPLABV3_PLUS_PATH - MLC_DOWNLOAD_URL: 'mlc-cognata:mlc_cognata_dataset/<<>>' - MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' extra_cache_tags: ml,model,deeplabv3,plus force_cache: true names: - - dae + - dae-deplabv3+ tags: download-and-extract force_env_keys: - MLC_OUTDIRNAME @@ -72,10 +83,27 @@ variations: MLC_DOWNLOAD_SRC: mlcommons rclone: group: download-tool + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config + force_cache: true + names: + - rclone-config + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes add_deps_recursive: - dae: + dae-deplabv3+: tags: _rclone default: true + r2-downloader: + group: download-tool + add_deps_recursive: + dae-deplabv3+: + tags: _r2-downloader dry-run: group: run-mode env: @@ -83,6 +111,9 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x tests: needs_pat: true run_inputs: @@ -90,3 +121,6 @@ tests: - onnx,rclone,mlc,dry-run - onnx_dynamic,rclone,mlc,dry-run - pytorch,rclone,mlc,dry-run + - onnx,r2-downloader,mlc,dry-run + - onnx_dynamic,r2-downloader,mlc,dry-run + - pytorch,r2-downloader,mlc,dry-run From 1f3cce5caaf0af43d0775295829694451efdf9a1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Oct 2025 14:30:45 +0000 Subject: [PATCH 23/55] [Automated Commit] Document script/get-ml-model-deeplabv3_plus/meta.yaml [skip ci] --- script/get-ml-model-deeplabv3_plus/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/script/get-ml-model-deeplabv3_plus/README.md b/script/get-ml-model-deeplabv3_plus/README.md index 438f35648..63a6cce07 100644 --- a/script/get-ml-model-deeplabv3_plus/README.md +++ b/script/get-ml-model-deeplabv3_plus/README.md @@ -63,6 +63,7 @@ No script specific inputs ### Download-tool +- `r2-downloader` - `rclone` (default) ### Model-format From b9f7bb2ca076000b0c2bc38df899328463ee67d1 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 10 Oct 2025 00:36:04 +0530 Subject: [PATCH 24/55] Use r2-downloader for cognata dataset (#668) --- .../meta.yaml | 61 +++++++++++++++---- .../get-preprocessed-dataset-cognata/run.sh | 2 +- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/script/get-preprocessed-dataset-cognata/meta.yaml b/script/get-preprocessed-dataset-cognata/meta.yaml index 940351310..c33caf57a 100644 --- a/script/get-preprocessed-dataset-cognata/meta.yaml +++ b/script/get-preprocessed-dataset-cognata/meta.yaml @@ -38,22 +38,42 @@ variations: env: MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAME: val_2d MLC_DATASET_COGNATA_TAR_FILENAME: val_2d.tar.gz + validation,2d_obj_det,rclone: + env: MLC_DOWNLOAD_URL: mlc-cognata:mlc_cognata_dataset/preprocessed_2d/<<>> + validation,2d_obj_det,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/val_2d.uri calibration,2d_obj_det: env: MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAME: calib_2d MLC_DATASET_COGNATA_TAR_FILENAME: calib_2d.tar.gz + calibration,2d_obj_det,rclone: + env: MLC_DOWNLOAD_URL: mlc-cognata:mlc_cognata_dataset/preprocessed_2d/<<>> + calibration,2d_obj_det,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/calib_2d.uri validation,segmentation: env: MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAME: val_seg MLC_DATASET_COGNATA_TAR_FILENAME: val_seg.tar.gz + validation,segmentation,rclone: + env: MLC_DOWNLOAD_URL: mlc-cognata:mlc_cognata_dataset/preprocessed_seg/<<>> + validation,segmentation,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/val_seg.uri calibration,segmentation: env: MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAME: calib_seg MLC_DATASET_COGNATA_TAR_FILENAME: calib_seg.tar.gz + calibration,segmentation,rclone: + env: MLC_DOWNLOAD_URL: mlc-cognata:mlc_cognata_dataset/preprocessed_seg/<<>> + calibration,segmentation,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/calib_seg.uri prebuilt: default: true group: dataset-src @@ -61,22 +81,13 @@ variations: MLC_NUSCENES_DATASET_TYPE: prebuilt MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_COGNATA_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_COGNATA_PATH + prebuilt,rclone: + env: MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' mlc: group: download-src default: true prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_config-name.mlc-cognata - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - env: - MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' @@ -92,6 +103,19 @@ variations: - MLC_DOWNLOAD_URL env: MLC_DOWNLOAD_SRC: mlcommons + mlc,rclone: + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_config-name.mlc-cognata + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 rclone: group: download-tool add_deps_recursive: @@ -105,10 +129,23 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + r2-downloader: + group: download-tool + add_deps_recursive: + dae: + tags: _r2-downloader + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x tests: + needs_pat: true run_inputs: - variations_list: - validation,prebuilt,2d_obj_det,rclone,mlc,dry-run - calibration,prebuilt,2d_obj_det,rclone,mlc,dry-run - validation,prebuilt,segmentation,rclone,mlc,dry-run - - calibration,prebuilt,segmentation,rclone,mlc,dry-run \ No newline at end of file + - calibration,prebuilt,segmentation,rclone,mlc,dry-run + - validation,prebuilt,2d_obj_det,r2-downloader,mlc,dry-run + - calibration,prebuilt,2d_obj_det,r2-downloader,mlc,dry-run + - validation,prebuilt,segmentation,r2-downloader,mlc,dry-run + - calibration,prebuilt,segmentation,r2-downloader,mlc,dry-run diff --git a/script/get-preprocessed-dataset-cognata/run.sh b/script/get-preprocessed-dataset-cognata/run.sh index 0c141b8e6..4e9dbe810 100644 --- a/script/get-preprocessed-dataset-cognata/run.sh +++ b/script/get-preprocessed-dataset-cognata/run.sh @@ -3,7 +3,7 @@ if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" ]]; then cd "${MLC_PREPROCESSED_DATASET_COGNATA_PATH}" || exit for f in *.tar.gz; do - tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + tar --no-same-owner -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } done cd - || exit fi \ No newline at end of file From 0ec5f59cc2e6212fbef58a53d9f0e1a0a2a9bb04 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Oct 2025 19:06:34 +0000 Subject: [PATCH 25/55] [Automated Commit] Document script/get-preprocessed-dataset-cognata/meta.yaml [skip ci] --- .../README.md | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 script/get-preprocessed-dataset-cognata/README.md diff --git a/script/get-preprocessed-dataset-cognata/README.md b/script/get-preprocessed-dataset-cognata/README.md new file mode 100644 index 000000000..84273df95 --- /dev/null +++ b/script/get-preprocessed-dataset-cognata/README.md @@ -0,0 +1,85 @@ +# README for get-preprocessed-dataset-cognata +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,dataset,cognata,preprocessed +``` + +No script specific inputs +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Dataset-src + +- `prebuilt` (default) + +### Dataset-type + +- `calibration` +- `validation` (default) + +### Download-src + +- `mlc` (default) + +### Download-tool + +- `r2-downloader` +- `rclone` (default) + +### Run-mode + +- `dry-run` + +### Task + +- `2d_obj_det` (default) +- `segmentation` From d5fd92bff35bc95122b97a440ccc2160501ec624 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 12 Oct 2025 20:49:01 +0100 Subject: [PATCH 26/55] Support remote-run script action (WIP) (#673) * Fix typo on install-llvm-src * Added select_script function --- automation/script/docker.py | 675 ++++++++++++++++---------------- automation/script/module.py | 45 +++ automation/script/remote_run.py | 231 +++++++++++ 3 files changed, 614 insertions(+), 337 deletions(-) create mode 100644 automation/script/remote_run.py diff --git a/automation/script/docker.py b/automation/script/docker.py index 7bf96b8b9..0d4aac13b 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -22,16 +22,7 @@ def dockerfile(self_module, input_params): is_quiet_mode = input_params.get('quiet', False) is_console_output = input_params.get('out') == 'con' - # Step 2: Search for scripts - search_result = self_module.search(input_params.copy()) - if search_result['return'] > 0: - return search_result - - scripts_list = search_result['list'] - if not scripts_list: - return {'return': 1, 'error': 'No scripts were found'} - - # Step 3: Process Dockerfile-related configurations + # Step 2: Process Dockerfile-related configurations env = input_params.get('env', {}) state_data = input_params.get('state', {}) constant_vars = input_params.get('const', {}) @@ -39,190 +30,199 @@ def dockerfile(self_module, input_params): tag_values = input_params.get('tags', '').split(",") variation_tags = [tag[1:] for tag in tag_values if tag.startswith("_")] - # Step 4: Iterate over scripts and generate Dockerfile - for script in sorted(scripts_list, key=lambda x: x.meta.get('alias', '')): - metadata = script.meta - script_directory = script.path - script_tags = metadata.get("tags", []) - script_alias = metadata.get('alias', '') - script_uid = metadata.get('uid', '') - - run_state = { - 'deps': [], - 'fake_deps': [], - 'parent': None, - 'script_id': f"{script_alias},{script_uid}", - 'script_variation_tags': variation_tags - } - docker_settings = metadata.get('docker', {}) - docker_settings_default_env = docker_settings.get('default_env', {}) - for key in docker_settings_default_env: - env.setdefault(key, docker_settings_default_env[key]) - - state_data['docker'] = docker_settings - add_deps_recursive = input_params.get('add_deps_recursive', {}) - - # Update state with metadata and variations - update_state_result = self_module.update_state_from_meta( - metadata, env, state_data, constant_vars, constant_state, - deps=[], - post_deps=[], - prehook_deps=[], - posthook_deps=[], - new_env_keys=[], - new_state_keys=[], - run_state=run_state, - i=input_params - ) - if update_state_result['return'] > 0: - return update_state_result - - update_variations_result = self_module._update_state_from_variations( - input_params, metadata, variation_tags, metadata.get( - 'variations', {}), - env, state_data, constant_vars, constant_state, - deps=[], # Add your dependencies if needed - post_deps=[], # Add post dependencies if needed - prehook_deps=[], # Add prehook dependencies if needed - posthook_deps=[], # Add posthook dependencies if needed - new_env_keys_from_meta=[], # Add keys from meta if needed - new_state_keys_from_meta=[], # Add state keys from meta if needed - add_deps_recursive=add_deps_recursive, - run_state=run_state, - recursion_spaces='' - ) - if update_variations_result['return'] > 0: - return update_variations_result - - # Set Docker-specific configurations - docker_settings = state_data['docker'] - - if is_true(docker_settings.get('pass_docker_to_script', False)): - input_params['docker'] = True - r = self_module.run(input_params) + r = self_module._select_script(input_params) + if r['return'] > 0: + return r + + script = r['script'] + + if not script: + return {'return': 1, + 'error': 'No scripts were found for generating dockerfile'} + + metadata = script.meta + script_directory = script.path + script_tags = metadata.get("tags", []) + script_alias = metadata.get('alias', '') + script_uid = metadata.get('uid', '') + + run_state = { + 'deps': [], + 'fake_deps': [], + 'parent': None, + 'script_id': f"{script_alias},{script_uid}", + 'script_variation_tags': variation_tags + } + + docker_settings = metadata.get('docker', {}) + docker_settings_default_env = docker_settings.get('default_env', {}) + for key in docker_settings_default_env: + env.setdefault(key, docker_settings_default_env[key]) + + state_data['docker'] = docker_settings + add_deps_recursive = input_params.get('add_deps_recursive', {}) + + # Update state with metadata and variations + update_state_result = self_module.update_state_from_meta( + metadata, env, state_data, constant_vars, constant_state, + deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys=[], + new_state_keys=[], + run_state=run_state, + i=input_params + ) + if update_state_result['return'] > 0: + return update_state_result + + update_variations_result = self_module._update_state_from_variations( + input_params, metadata, variation_tags, metadata.get( + 'variations', {}), + env, state_data, constant_vars, constant_state, + deps=[], # Add your dependencies if needed + post_deps=[], # Add post dependencies if needed + prehook_deps=[], # Add prehook dependencies if needed + posthook_deps=[], # Add posthook dependencies if needed + new_env_keys_from_meta=[], # Add keys from meta if needed + new_state_keys_from_meta=[], # Add state keys from meta if needed + add_deps_recursive=add_deps_recursive, + run_state=run_state, + recursion_spaces='' + ) + if update_variations_result['return'] > 0: + return update_variations_result + + # Set Docker-specific configurations + docker_settings = state_data['docker'] + + if is_true(docker_settings.get('pass_docker_to_script', False)): + input_params['docker'] = True + r = self_module.run(input_params) + return r + + if not docker_settings.get('run', True) and not input_params.get( + 'docker_run_override', False): + logger.info("Docker 'run' is set to False in meta.json") + return {'return': 0, 'warning': 'Docker run is set to false in script meta'} + + # Handle build dependencies + show_time = input_params.get('show_time', False) + deps = docker_settings.get('build_deps', []) + if deps: + r = self_module._run_deps( + deps, [], env, {}, {}, {}, add_deps_recursive, '', [], '', False, '', + show_time, ' ', run_state) + if r['return'] > 0: return r - if not docker_settings.get('run', True) and not input_params.get( - 'docker_run_override', False): - logger.info("Docker 'run' is set to False in meta.json") - continue - - # Handle build dependencies - show_time = input_params.get('show_time', False) - deps = docker_settings.get('build_deps', []) - if deps: - r = self_module._run_deps( - deps, [], env, {}, {}, {}, add_deps_recursive, '', [], '', False, '', - show_time, ' ', run_state) - if r['return'] > 0: - return r - - update_state_result = self_module.update_state_from_meta( - metadata, env, state_data, constant_vars, constant_state, - deps=[], - post_deps=[], - prehook_deps=[], - posthook_deps=[], - new_env_keys=[], - new_state_keys=[], - run_state=run_state, - i=input_params - ) - if update_state_result['return'] > 0: - return update_state_result - docker_settings = state_data['docker'] - - # Prune temporary environment variables - run_command = copy.deepcopy(run_command_arc) - for key in list(run_command.get('env', {}).keys()): - if key.startswith("MLC_TMP_"): - del run_command['env'][key] - - # Regenerate script command - regenerate_result = regenerate_script_cmd({ - 'script_uid': script_uid, - 'script_alias': script_alias, - 'run_cmd': run_command, - 'tags': script_tags, - 'fake_run': True, - 'docker_settings': docker_settings, - 'docker_run_cmd_prefix': input_params.get('docker_run_cmd_prefix', docker_settings.get('run_cmd_prefix', '')) - }) - if regenerate_result['return'] > 0: - return regenerate_result - - run_command_string = regenerate_result['run_cmd_string'] - - # Prepare Docker-specific inputs - docker_inputs, dockerfile_path = prepare_docker_inputs( - input_params, docker_settings, script_directory) - - # Handle optional dependencies and comments - if input_params.get('print_deps'): - mlc_input = { - 'action': 'run', 'automation': 'script', 'tags': input_params.get('tags'), - 'print_deps': True, 'quiet': True, 'silent': True, - 'fake_run': True, 'fake_deps': True - } - deps_result = self_module.action_object.access(mlc_input) - if deps_result['return'] > 0: - return deps_result - comments = [ - f"#RUN {dep}" for dep in deps_result['new_state']['print_deps']] - else: - comments = [] - - # Push Docker image if specified - if input_params.get('docker_push_image') in [True, 'True', 'yes']: - env['MLC_DOCKER_PUSH_IMAGE'] = 'yes' - - dockerfile_env = docker_inputs.get('env', {}) - dockerfile_build_env = docker_inputs.get('build_env', {}) - - dockerfile_env['MLC_RUN_STATE_DOCKER'] = True - # Generate Dockerfile - mlc_docker_input = { - 'action': 'run', 'automation': 'script', 'tags': 'build,dockerfile', - 'fake_run_option': " " if docker_inputs.get('real_run') else " --fake_run", - 'comments': comments, 'run_cmd': f"{run_command_string} --quiet", - 'script_tags': input_params.get('tags'), 'env': env, - 'dockerfile_env': dockerfile_env, - 'dockerfile_build_env': dockerfile_build_env, - 'quiet': True, 'real_run': True + update_state_result = self_module.update_state_from_meta( + metadata, env, state_data, constant_vars, constant_state, + deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys=[], + new_state_keys=[], + run_state=run_state, + i=input_params + ) + if update_state_result['return'] > 0: + return update_state_result + docker_settings = state_data['docker'] + + # Prune temporary environment variables + run_command = copy.deepcopy(run_command_arc) + for key in list(run_command.get('env', {}).keys()): + if key.startswith("MLC_TMP_"): + del run_command['env'][key] + + # Regenerate script command + regenerate_result = regenerate_script_cmd({ + 'script_uid': script_uid, + 'script_alias': script_alias, + 'run_cmd': run_command, + 'tags': script_tags, + 'fake_run': True, + 'docker_settings': docker_settings, + 'docker_run_cmd_prefix': input_params.get('docker_run_cmd_prefix', docker_settings.get('run_cmd_prefix', '')) + }) + if regenerate_result['return'] > 0: + return regenerate_result + + run_command_string = regenerate_result['run_cmd_string'] + + # Prepare Docker-specific inputs + docker_inputs, dockerfile_path = prepare_docker_inputs( + input_params, docker_settings, script_directory) + + # Handle optional dependencies and comments + if input_params.get('print_deps'): + mlc_input = { + 'action': 'run', 'automation': 'script', 'tags': input_params.get('tags'), + 'print_deps': True, 'quiet': True, 'silent': True, + 'fake_run': True, 'fake_deps': True } - - docker_v = False + deps_result = self_module.action_object.access(mlc_input) + if deps_result['return'] > 0: + return deps_result + comments = [ + f"#RUN {dep}" for dep in deps_result['new_state']['print_deps']] + else: + comments = [] + + # Push Docker image if specified + if input_params.get('docker_push_image') in [True, 'True', 'yes']: + env['MLC_DOCKER_PUSH_IMAGE'] = 'yes' + + dockerfile_env = docker_inputs.get('env', {}) + dockerfile_build_env = docker_inputs.get('build_env', {}) + + dockerfile_env['MLC_RUN_STATE_DOCKER'] = True + # Generate Dockerfile + mlc_docker_input = { + 'action': 'run', 'automation': 'script', 'tags': 'build,dockerfile', + 'fake_run_option': " " if docker_inputs.get('real_run') else " --fake_run", + 'comments': comments, 'run_cmd': f"{run_command_string} --quiet", + 'script_tags': input_params.get('tags'), 'env': env, + 'dockerfile_env': dockerfile_env, + 'dockerfile_build_env': dockerfile_build_env, + 'quiet': True, 'real_run': True + } + + docker_v = False + docker_s = False + if is_true(input_params.get( + 'docker_v', input_params.get('docker_verbose', False))): + docker_v = True + if is_true(input_params.get( + 'docker_s', input_params.get('docker_silent', False))): + docker_s = True + + if docker_s and docker_v: + logger.warning( + "Both verbose and silent is set to True. Verbose will take precedence.") docker_s = False - if is_true(input_params.get( - 'docker_v', input_params.get('docker_verbose', False))): + + if not docker_s and not docker_v: + if logger.level == logging.DEBUG: docker_v = True - if is_true(input_params.get( - 'docker_s', input_params.get('docker_silent', False))): + elif logger.level == logging.WARNING: docker_s = True - if docker_s and docker_v: - logger.warning( - "Both verbose and silent is set to True. Verbose will take precedence.") - docker_s = False - - if not docker_s and not docker_v: - if logger.level == logging.DEBUG: - docker_v = True - elif logger.level == logging.WARNING: - docker_s = True + if docker_s: + mlc_docker_input['run_cmd'] += ' -s' + elif docker_v: + mlc_docker_input['run_cmd'] += ' -v' - if docker_s: - mlc_docker_input['run_cmd'] += ' -s' - elif docker_v: - mlc_docker_input['run_cmd'] += ' -v' + mlc_docker_input.update(docker_inputs) - mlc_docker_input.update(docker_inputs) + dockerfile_result = self_module.action_object.access(mlc_docker_input) + if dockerfile_result['return'] > 0: + return dockerfile_result - dockerfile_result = self_module.action_object.access(mlc_docker_input) - if dockerfile_result['return'] > 0: - return dockerfile_result - - logger.info(f"Dockerfile generated at {dockerfile_path}") + logger.info(f"Dockerfile generated at {dockerfile_path}") return {'return': 0} @@ -256,13 +256,6 @@ def docker_run(self_module, i): # Save current directory and prepare to search for scripts cur_dir = os.getcwd() - r = self_module.search(i.copy()) - if r['return'] > 0: - return r - - lst = r['list'] - if not lst: - return {'return': 1, 'error': 'No scripts were found'} env['MLC_RUN_STATE_DOCKER'] = False state, const, const_state = i.get( @@ -281,165 +274,173 @@ def docker_run(self_module, i): input_i = copy.deepcopy(i) - # Process each artifact - for artifact in sorted(lst, key=lambda x: x.meta.get('alias', '')): - meta, script_path = artifact.meta, artifact.path - tags, script_alias, script_uid = meta.get( - "tags", []), meta.get( - 'alias', ''), meta.get( - 'uid', '') - - mounts = copy.deepcopy( - i.get( - 'docker_mounts', - [])) # do we need a copy here? - variations = meta.get('variations', {}) - - # take the folder path as well as file path env variables from meta - file_path_env_keys = meta.get('file_path_env_keys', []) - folder_path_env_keys = meta.get('folder_path_env_keys', []) - - docker_settings = meta.get('docker', {}) - docker_settings_default_env = docker_settings.get('default_env', {}) - for key in docker_settings_default_env: - env.setdefault(key, docker_settings_default_env[key]) - - state['docker'] = docker_settings - run_state = { - 'deps': [], 'fake_deps': [], 'parent': None, - 'script_id': f"{script_alias},{script_uid}", - 'script_variation_tags': variation_tags, - 'file_path_env_keys': file_path_env_keys, - 'folder_path_env_keys': folder_path_env_keys - } + r = self_module._select_script(i) + if r['return'] > 0: + return r - # Update state and handle variations - r = self_module.update_state_from_meta(meta, env, state, const, const_state, deps=[], - post_deps=[], - prehook_deps=[], - posthook_deps=[], - new_env_keys=[], - new_state_keys=[], run_state=run_state, i=i) - if r['return'] > 0: - return r + script = r['script'] + + if not script: + return {'return': 1, + 'error': 'No scripts were found for generating dockerfile'} + + meta, script_path = script.meta, script.path + tags, script_alias, script_uid = meta.get( + "tags", []), meta.get( + 'alias', ''), meta.get( + 'uid', '') + + mounts = copy.deepcopy( + i.get( + 'docker_mounts', + [])) # do we need a copy here? + variations = meta.get('variations', {}) + + # take the folder path as well as file path env variables from meta + file_path_env_keys = meta.get('file_path_env_keys', []) + folder_path_env_keys = meta.get('folder_path_env_keys', []) + + docker_settings = meta.get('docker', {}) + docker_settings_default_env = docker_settings.get('default_env', {}) + for key in docker_settings_default_env: + env.setdefault(key, docker_settings_default_env[key]) + + state['docker'] = docker_settings + run_state = { + 'deps': [], 'fake_deps': [], 'parent': None, + 'script_id': f"{script_alias},{script_uid}", + 'script_variation_tags': variation_tags, + 'file_path_env_keys': file_path_env_keys, + 'folder_path_env_keys': folder_path_env_keys + } + + # Update state and handle variations + r = self_module.update_state_from_meta(meta, env, state, const, const_state, deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys=[], + new_state_keys=[], run_state=run_state, i=i) + if r['return'] > 0: + return r - r = self_module._update_state_from_variations( - i, meta, variation_tags, variations, env, state, const, const_state, deps=[], - post_deps=[], - prehook_deps=[], - posthook_deps=[], - new_env_keys_from_meta=[], - new_state_keys_from_meta=[], - add_deps_recursive=add_deps_recursive, run_state=run_state, recursion_spaces='') - if r['return'] > 0: - return r + r = self_module._update_state_from_variations( + i, meta, variation_tags, variations, env, state, const, const_state, deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys_from_meta=[], + new_state_keys_from_meta=[], + add_deps_recursive=add_deps_recursive, run_state=run_state, recursion_spaces='') + if r['return'] > 0: + return r - docker_settings = state['docker'] - - deps = docker_settings.get('deps', []) - if deps: - r = self_module._run_deps( - deps, [], env, {}, {}, {}, add_deps_recursive, '', [], '', False, '', - show_time, ' ', run_state) - if r['return'] > 0: - return r - - # For updating meta from update_meta_if_env - r = self_module.update_state_from_meta( - meta, env, state, const, const_state, deps=[], - post_deps=[], - prehook_deps=[], - posthook_deps=[], - new_env_keys=[], - new_state_keys=[], - run_state=run_state, - i=i) + docker_settings = state['docker'] + + deps = docker_settings.get('deps', []) + if deps: + r = self_module._run_deps( + deps, [], env, {}, {}, {}, add_deps_recursive, '', [], '', False, '', + show_time, ' ', run_state) if r['return'] > 0: return r - # Skip scripts marked as non-runnable - if not docker_settings.get('run', True) and not i.get( - 'docker_run_override', False): - logger.info("docker.run set to False in meta.yaml") - continue + # For updating meta from update_meta_if_env + r = self_module.update_state_from_meta( + meta, env, state, const, const_state, deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys=[], + new_state_keys=[], + run_state=run_state, + i=i) + if r['return'] > 0: + return r - if is_true(docker_settings.get('pass_docker_to_script', False)): - logger.info("Docker 'run' is passed to the run script") - i['docker'] = True - r = self_module.run(i) - return r + # Skip scripts marked as non-runnable + if not docker_settings.get('run', True) and not i.get( + 'docker_run_override', False): + logger.info("docker.run set to False in meta.yaml") + return {'return': 0, 'warning': 'Docker run is set to false in script meta'} - # Regenerate Dockerfile if required - if regenerate_docker_file: - r = dockerfile(self_module, input_i) - if r['return'] > 0: - return r + if is_true(docker_settings.get('pass_docker_to_script', False)): + logger.info("Docker 'run' is passed to the run script") + i['docker'] = True + r = self_module.run(i) + return r - # Ensure Docker is available - r = self_module.action_object.access( - {'action': 'run', 'automation': 'script', 'tags': "get,docker"}) + # Regenerate Dockerfile if required + if regenerate_docker_file: + r = dockerfile(self_module, input_i) if r['return'] > 0: return r - r = self_module._update_env_from_input(env, i) - if r['return'] > 0: - return r + # Ensure Docker is available + r = self_module.action_object.access( + {'action': 'run', 'automation': 'script', 'tags': "get,docker"}) + if r['return'] > 0: + return r - # Prepare Docker-specific inputs - docker_inputs, dockerfile_path = prepare_docker_inputs( - i, docker_settings, script_path, True) - - if docker_inputs is None: - return {'return': 1, 'error': 'Error preparing Docker inputs'} - - docker_input_mapping = docker_settings.get('input_mapping') - - # Update env based on docker_input_mapping if they are in input - if docker_input_mapping and i: - env.update({docker_input_mapping[key]: i[key] - for key in docker_input_mapping if key in i}) - - if docker_inputs.get('user'): - docker_settings['user'] = docker_inputs['user'] - - # Handle environment variable-based mounts - res = process_mounts( - mounts, - env, - docker_settings, - f_run_cmd, - run_state) - if res['return'] > 0: - return res - docker_inputs['mounts'] = res['mounts'] - container_env_string = res['container_env_string'] - - res = update_docker_environment( - docker_settings, env, self_module.host_env_keys, container_env_string) - if res['return'] > 0: - return res - - # Generate the run command - r = regenerate_script_cmd({'script_uid': script_uid, - 'script_alias': script_alias, - 'tags': tags, - 'run_cmd': f_run_cmd}) - if r['return'] > 0: - return r - final_run_cmd = f"""{r['run_cmd_string']} {container_env_string} --docker_run_deps """ - - # Execute the Docker container - mlc_docker_input = { - 'action': 'run', 'target': 'script', 'tags': 'run,docker,container', - 'rebuild': rebuild_docker_image, - 'env': env, 'mounts': mounts, - 'script_tags': i.get('tags'), 'run_cmd': final_run_cmd, - 'quiet': True, 'real_run': True, 'add_deps_recursive': {'build-docker-image': {'dockerfile': dockerfile_path}}, - **docker_inputs - } + r = self_module._update_env_from_input(env, i) + if r['return'] > 0: + return r - r = self_module.action_object.access(mlc_docker_input) - if r['return'] > 0: - return r + # Prepare Docker-specific inputs + docker_inputs, dockerfile_path = prepare_docker_inputs( + i, docker_settings, script_path, True) + + if docker_inputs is None: + return {'return': 1, 'error': 'Error preparing Docker inputs'} + + docker_input_mapping = docker_settings.get('input_mapping') + + # Update env based on docker_input_mapping if they are in input + if docker_input_mapping and i: + env.update({docker_input_mapping[key]: i[key] + for key in docker_input_mapping if key in i}) + + if docker_inputs.get('user'): + docker_settings['user'] = docker_inputs['user'] + + # Handle environment variable-based mounts + res = process_mounts( + mounts, + env, + docker_settings, + f_run_cmd, + run_state) + if res['return'] > 0: + return res + docker_inputs['mounts'] = res['mounts'] + container_env_string = res['container_env_string'] + + res = update_docker_environment( + docker_settings, env, self_module.host_env_keys, container_env_string) + if res['return'] > 0: + return res + + # Generate the run command + r = regenerate_script_cmd({'script_uid': script_uid, + 'script_alias': script_alias, + 'tags': tags, + 'run_cmd': f_run_cmd}) + if r['return'] > 0: + return r + final_run_cmd = f"""{r['run_cmd_string']} {container_env_string} --docker_run_deps """ + + # Execute the Docker container + mlc_docker_input = { + 'action': 'run', 'target': 'script', 'tags': 'run,docker,container', + 'rebuild': rebuild_docker_image, + 'env': env, 'mounts': mounts, + 'script_tags': i.get('tags'), 'run_cmd': final_run_cmd, + 'quiet': True, 'real_run': True, 'add_deps_recursive': {'build-docker-image': {'dockerfile': dockerfile_path}}, + **docker_inputs + } + + r = self_module.action_object.access(mlc_docker_input) + if r['return'] > 0: + return r return {'return': 0} diff --git a/automation/script/module.py b/automation/script/module.py index 9be00489e..e44d6db91 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4592,6 +4592,11 @@ def experiment(self, i): from script.experiment import experiment_run return experiment_run(self, i) + ############################################################ + def remote_run(self, i): + from script.remote_run import remote_run + return remote_run(self, i) + ########################################################################## def _available_variations(self, i): @@ -4651,6 +4656,46 @@ def clean_some_tmp_files(self, i): def _get_script_name(self, env, path, filename="run"): return get_script_name(env, path, filename) + def _select_script(self, i): + r = self.search(i.copy()) + if r['return'] > 0: + return r + + lst = r['list'] + if not lst: + return {'return': 1, 'error': 'No scripts were found'} + + # Sort scripts by alias for consistent display + sorted_list = sorted(lst, key=lambda x: x.meta.get('alias', '')) + + # If quiet mode is off, prompt the user + if not i.get('quiet', False): + print("\nAvailable scripts:") + for idx, artifact in enumerate(sorted_list, 1): + meta = artifact.meta + alias = meta.get('alias', '') + uid = meta.get('uid', '') + tags = ', '.join(meta.get('tags', [])) + print(f"{idx}. {alias or uid} [{tags}]") + + try: + choice = int(input("\nSelect a script by number: ")) - 1 + except ValueError: + return {'return': 1, 'error': 'Invalid selection'} + + if choice < 0 or choice >= len(sorted_list): + return {'return': 1, 'error': 'Selection out of range'} + + selected_artifact = sorted_list[choice] + else: + # Quiet mode: select the first script + selected_artifact = sorted_list[0] + + return { + 'return': 0, + 'script': selected_artifact + } + def get_version_tag_from_version(version, cached_tags): tags_to_add = [] diff --git a/automation/script/remote_run.py b/automation/script/remote_run.py new file mode 100644 index 000000000..f8d7b9ae3 --- /dev/null +++ b/automation/script/remote_run.py @@ -0,0 +1,231 @@ +from collections import defaultdict +import os +from mlc.main import ExperimentAction +import mlc.utils as utils +from mlc import utils +from utils import * +import logging +from pathlib import PureWindowsPath, PurePosixPath +import time +import copy +from datetime import datetime + + +def remote_run(self_module, i): + """ + Remote run of MLC scripts. + + Args: + self_module: Reference to the current module for internal calls. + i: Dictionary containing input parameters for the experiment execution. + + Returns: + Dictionary with the result of the operation. Keys: + - 'return': 0 on success, >0 on error. + - 'error': Error message (if any). + """ + + # Extract and handle basic inputs + quiet = i.get('quiet', False) + show_time = i.get('show_time', False) + logger = self_module.logger + env = i.get('env', {}) + remote_host = i.get('remote_host', 'localhost') + remote_port = i.get('remote_port', 22) + + prune_result = prune_input( + {'input': i, 'extra_keys_starts_with': ['remote_']}) + if prune_result['return'] > 0: + return prune_result + + run_input = prune_result['new_input'] + + r = convert_input(i) + + cur_dir = os.getcwd() + r = self_module.search(i.copy()) + if r['return'] > 0: + return r + + lst = r['list'] + if not lst: + return {'return': 1, 'error': 'No scripts were found'} + + # Process each artifact + for artifact in sorted(lst, key=lambda x: x.meta.get('alias', '')): + meta, script_path = artifact.meta, artifact.path + tags, script_alias, script_uid = meta.get( + "tags", []), meta.get( + 'alias', ''), meta.get( + 'uid', '') + + # Execute the experiment script + mlc_script_input = { + 'action': 'run', 'target': 'script' + } + if exp: + for key in exp: + ii = {**mlc_script_input, **run_input} + if isinstance(exp[key], list): + for val in exp[key]: + ii[key] = val + r = run_script_and_tag_experiment( + ii, + self_module.action_object, + experiment_action, + tags, + extra_exp_tags, + meta, + skip_state_save, + logger) + if r['return'] > 0: + return r + elif isinstance(exp[key], dict): + return { + 'return': 1, 'error': 'Dictionary inputs are not supported for mlc experiment script'} + else: + ii[key] = exp[key] + r = run_script_and_tag_experiment( + ii, + self_module.action_object, + experiment_action, + tags, + extra_exp_tags, + meta, + skip_state_save, + logger) + if r['return'] > 0: + return r + + return {'return': 0} + + +def run_script_and_tag_experiment( + ii, script_action, experiment_action, tags, extra_exp_tags, script_meta, skip_state_save, logger): + + current_path = os.path.abspath(os.getcwd()) + experiment_meta = {} + recursion_spaces = '' + exp_tags = tags + extra_exp_tags + ssi = {'action': 'update', + 'target': 'experiment', + 'script_alias': script_meta['alias'], + 'script_uid': script_meta['uid'], + 'tags': ','.join(exp_tags), + 'extra_tags': ",".join(extra_exp_tags), + 'meta': experiment_meta, + 'force': True} + + r = experiment_action.access(ssi) + if r['return'] > 0: + return r + + experiment = r['list'][0] + + logger.debug( + recursion_spaces + + ' - Changing to {}'.format(experiment.path)) + + os.chdir(experiment.path) + # Get current datetime in YYYY-MM-DD_HH-MM-SS format + timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + + # Create a folder name using the timestamp + folder_name = f"run_{timestamp}" + + # Create the directory + os.makedirs(folder_name, exist_ok=True) + os.chdir(folder_name) + + if not skip_state_save: + ssi = {'action': 'run', + 'target': 'script', + 'tags': 'save,system,state', + 'outfile': 'system_state_before.json', + 'quiet': True + } + r = script_action.access(ssi) + if r['return'] > 0: + return r + + start_time = time.time() + r = script_action.access(ii) + if r['return'] > 0: + return r + + end_time = time.time() + elapsed = end_time - start_time + time_taken_string = format_elapsed(elapsed) + logger.info(f"Time taken: {time_taken_string}") + + if not skip_state_save: + ssi['outfile'] = 'system_state_after.json' + r = script_action.access(ssi) + if r['return'] > 0: + return r + + ''' + exp_tags = tags + ii = {'action': 'update', + 'target': 'experiment', + 'uid': experiment.meta['uid'], + 'meta': experiment.meta, + 'script_alias': meta['alias'], + 'replace_lists': True, # To replace tags + 'tags': ','.join(exp_tags)} + + r = experiment_action.access(ii) + if r['return'] > 0: + return r + ''' + os.chdir(current_path) + logger.info( + f"Experiment entry saved at: {os.path.join(experiment.path, folder_name)}") + + return {'return': 0, 'experiment': experiment, 'folder_name': folder_name} + + elif seconds < 3600: + mins, secs = divmod(seconds, 60) + return f"{int(mins)} minutes {secs:.1f} seconds" + else: + hours, remainder = divmod(seconds, 3600) + mins, secs = divmod(remainder, 60) + return f"{int(hours)} hours {int(mins)} minutes {secs:.1f} seconds" + + +def parse_value(val): + if isinstance(val, list): + return [parse_value(v) for v in val] + + val = str(val) + + # Handle range inputs like 2:10 or 2:10:2 + if ':' in val: + parts = val.split(':') + try: + parts = list(map(int, parts)) + if len(parts) == 2: + return list(range(parts[0], parts[1] + 1)) + elif len(parts) == 3: + return list(range(parts[0], parts[1] + 1, parts[2])) + except ValueError: + pass # Not a valid range, fall through + + # Convert to int if possible + if val.isdigit(): + return int(val) + + return val + + +def convert_input(input_dict): + output = defaultdict(dict) + + for key, value in input_dict.items(): + if '.' in key: + main_key, sub_key = key.split('.', 1) + output[main_key][sub_key] = parse_value(value) + elif isinstance(value, dict): + output[key].update({k: parse_value(v) for k, v in value.items()}) + + return dict(output) From 46d3511ee033816578515c04627031ae3cb095e8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 12 Oct 2025 22:54:52 +0100 Subject: [PATCH 27/55] Fixes for remote-run script (#674) * remote-run action now works --- automation/script/experiment.py | 75 ++++++----- automation/script/module.py | 2 +- automation/script/remote_run.py | 221 ++++++-------------------------- 3 files changed, 76 insertions(+), 222 deletions(-) diff --git a/automation/script/experiment.py b/automation/script/experiment.py index ac414fa4c..c5b63e5ed 100644 --- a/automation/script/experiment.py +++ b/automation/script/experiment.py @@ -50,48 +50,29 @@ def experiment_run(self_module, i): exp = {} cur_dir = os.getcwd() - r = self_module.search(i.copy()) + + r = self_module._select_script(i) if r['return'] > 0: return r - lst = r['list'] - if not lst: - return {'return': 1, 'error': 'No scripts were found'} - - # Process each artifact - for artifact in sorted(lst, key=lambda x: x.meta.get('alias', '')): - meta, script_path = artifact.meta, artifact.path - tags, script_alias, script_uid = meta.get( - "tags", []), meta.get( - 'alias', ''), meta.get( - 'uid', '') - - # Execute the experiment script - mlc_script_input = { - 'action': 'run', 'target': 'script' - } - if exp: - for key in exp: - ii = {**mlc_script_input, **run_input} - if isinstance(exp[key], list): - for val in exp[key]: - ii[key] = val - r = run_script_and_tag_experiment( - ii, - self_module.action_object, - experiment_action, - tags, - extra_exp_tags, - meta, - skip_state_save, - logger) - if r['return'] > 0: - return r - elif isinstance(exp[key], dict): - return { - 'return': 1, 'error': 'Dictionary inputs are not supported for mlc experiment script'} - else: - ii[key] = exp[key] + script = r['script'] + + meta, script_path = script.meta, script.path + tags, script_alias, script_uid = meta.get( + "tags", []), meta.get( + 'alias', ''), meta.get( + 'uid', '') + + # Execute the experiment script + mlc_script_input = { + 'action': 'run', 'target': 'script' + } + if exp: + for key in exp: + ii = {**mlc_script_input, **run_input} + if isinstance(exp[key], list): + for val in exp[key]: + ii[key] = val r = run_script_and_tag_experiment( ii, self_module.action_object, @@ -103,6 +84,22 @@ def experiment_run(self_module, i): logger) if r['return'] > 0: return r + elif isinstance(exp[key], dict): + return { + 'return': 1, 'error': 'Dictionary inputs are not supported for mlc experiment script'} + else: + ii[key] = exp[key] + r = run_script_and_tag_experiment( + ii, + self_module.action_object, + experiment_action, + tags, + extra_exp_tags, + meta, + skip_state_save, + logger) + if r['return'] > 0: + return r return {'return': 0} diff --git a/automation/script/module.py b/automation/script/module.py index e44d6db91..a815bdd38 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4669,7 +4669,7 @@ def _select_script(self, i): sorted_list = sorted(lst, key=lambda x: x.meta.get('alias', '')) # If quiet mode is off, prompt the user - if not i.get('quiet', False): + if not i.get('quiet', False) and len(sorted_list) > 1: print("\nAvailable scripts:") for idx, artifact in enumerate(sorted_list, 1): meta = artifact.meta diff --git a/automation/script/remote_run.py b/automation/script/remote_run.py index f8d7b9ae3..f0ea97aac 100644 --- a/automation/script/remote_run.py +++ b/automation/script/remote_run.py @@ -1,6 +1,5 @@ from collections import defaultdict import os -from mlc.main import ExperimentAction import mlc.utils as utils from mlc import utils from utils import * @@ -39,193 +38,51 @@ def remote_run(self_module, i): return prune_result run_input = prune_result['new_input'] - - r = convert_input(i) + mlc_run_cmd = run_input['mlc_run_cmd'] cur_dir = os.getcwd() - r = self_module.search(i.copy()) - if r['return'] > 0: - return r - - lst = r['list'] - if not lst: - return {'return': 1, 'error': 'No scripts were found'} - - # Process each artifact - for artifact in sorted(lst, key=lambda x: x.meta.get('alias', '')): - meta, script_path = artifact.meta, artifact.path - tags, script_alias, script_uid = meta.get( - "tags", []), meta.get( - 'alias', ''), meta.get( - 'uid', '') - - # Execute the experiment script - mlc_script_input = { - 'action': 'run', 'target': 'script' - } - if exp: - for key in exp: - ii = {**mlc_script_input, **run_input} - if isinstance(exp[key], list): - for val in exp[key]: - ii[key] = val - r = run_script_and_tag_experiment( - ii, - self_module.action_object, - experiment_action, - tags, - extra_exp_tags, - meta, - skip_state_save, - logger) - if r['return'] > 0: - return r - elif isinstance(exp[key], dict): - return { - 'return': 1, 'error': 'Dictionary inputs are not supported for mlc experiment script'} - else: - ii[key] = exp[key] - r = run_script_and_tag_experiment( - ii, - self_module.action_object, - experiment_action, - tags, - extra_exp_tags, - meta, - skip_state_save, - logger) - if r['return'] > 0: - return r - - return {'return': 0} - - -def run_script_and_tag_experiment( - ii, script_action, experiment_action, tags, extra_exp_tags, script_meta, skip_state_save, logger): - - current_path = os.path.abspath(os.getcwd()) - experiment_meta = {} - recursion_spaces = '' - exp_tags = tags + extra_exp_tags - ssi = {'action': 'update', - 'target': 'experiment', - 'script_alias': script_meta['alias'], - 'script_uid': script_meta['uid'], - 'tags': ','.join(exp_tags), - 'extra_tags': ",".join(extra_exp_tags), - 'meta': experiment_meta, - 'force': True} - r = experiment_action.access(ssi) + r = self_module._select_script(i) if r['return'] > 0: return r - experiment = r['list'][0] - - logger.debug( - recursion_spaces + - ' - Changing to {}'.format(experiment.path)) - - os.chdir(experiment.path) - # Get current datetime in YYYY-MM-DD_HH-MM-SS format - timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") - - # Create a folder name using the timestamp - folder_name = f"run_{timestamp}" - - # Create the directory - os.makedirs(folder_name, exist_ok=True) - os.chdir(folder_name) - - if not skip_state_save: - ssi = {'action': 'run', - 'target': 'script', - 'tags': 'save,system,state', - 'outfile': 'system_state_before.json', - 'quiet': True - } - r = script_action.access(ssi) - if r['return'] > 0: - return r - - start_time = time.time() - r = script_action.access(ii) + script = r['script'] + + meta, script_path = script.meta, script.path + tags, script_alias, script_uid = meta.get( + "tags", []), meta.get( + 'alias', ''), meta.get( + 'uid', '') + + # Execute the experiment script + mlc_script_input = { + 'action': 'run', 'target': 'script' + } + + run_cmds = [] + remote_mlc_python_venv = i.get('remote_python_venv', 'mlcflow') + run_cmds.append(f". {remote_mlc_python_venv}/bin/activate") + if i.get('remote_pull_mlc_repos', False): + run_cmds.append("mlc pull repo") + + script_run_cmd = " ".join(mlc_run_cmd.split(" ")[3:]) + run_cmds.append(f"mlcr {script_run_cmd}") + + remote_inputs = {} + for key in ["host", "port", "user", "client_refresh", + "password", "skip_host_verify", "ssh_key_file"]: + if i.get(f"remote_{key}"): + remote_inputs[key] = i[f"remote_{key}"] + + # Execute the remote command + mlc_remote_input = { + 'action': 'run', 'target': 'script', 'tags': 'remote,run,cmds,ssh', + 'script_tags': i.get('tags'), 'run_cmds': run_cmds, + **remote_inputs + } + + r = self_module.action_object.access(mlc_remote_input) if r['return'] > 0: return r - end_time = time.time() - elapsed = end_time - start_time - time_taken_string = format_elapsed(elapsed) - logger.info(f"Time taken: {time_taken_string}") - - if not skip_state_save: - ssi['outfile'] = 'system_state_after.json' - r = script_action.access(ssi) - if r['return'] > 0: - return r - - ''' - exp_tags = tags - ii = {'action': 'update', - 'target': 'experiment', - 'uid': experiment.meta['uid'], - 'meta': experiment.meta, - 'script_alias': meta['alias'], - 'replace_lists': True, # To replace tags - 'tags': ','.join(exp_tags)} - - r = experiment_action.access(ii) - if r['return'] > 0: - return r - ''' - os.chdir(current_path) - logger.info( - f"Experiment entry saved at: {os.path.join(experiment.path, folder_name)}") - - return {'return': 0, 'experiment': experiment, 'folder_name': folder_name} - - elif seconds < 3600: - mins, secs = divmod(seconds, 60) - return f"{int(mins)} minutes {secs:.1f} seconds" - else: - hours, remainder = divmod(seconds, 3600) - mins, secs = divmod(remainder, 60) - return f"{int(hours)} hours {int(mins)} minutes {secs:.1f} seconds" - - -def parse_value(val): - if isinstance(val, list): - return [parse_value(v) for v in val] - - val = str(val) - - # Handle range inputs like 2:10 or 2:10:2 - if ':' in val: - parts = val.split(':') - try: - parts = list(map(int, parts)) - if len(parts) == 2: - return list(range(parts[0], parts[1] + 1)) - elif len(parts) == 3: - return list(range(parts[0], parts[1] + 1, parts[2])) - except ValueError: - pass # Not a valid range, fall through - - # Convert to int if possible - if val.isdigit(): - return int(val) - - return val - - -def convert_input(input_dict): - output = defaultdict(dict) - - for key, value in input_dict.items(): - if '.' in key: - main_key, sub_key = key.split('.', 1) - output[main_key][sub_key] = parse_value(value) - elif isinstance(value, dict): - output[key].update({k: parse_value(v) for k, v in value.items()}) - - return dict(output) + return {'return': 0} From aadc967017f07ffd5124fa80e41cb03ca31ddec9 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 13 Oct 2025 21:04:14 +0530 Subject: [PATCH 28/55] Support deepseek-r1 model download through r2-downloader (#675) * Support deepseek-r1 model download * handle dry run --- .github/workflows/build_wheels.yml | 1 - script/get-ml-model-deepseek-r1/COPYRIGHT.md | 9 +++ script/get-ml-model-deepseek-r1/customize.py | 31 +++++++++++ script/get-ml-model-deepseek-r1/meta.yaml | 58 ++++++++++++++++++++ 4 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 script/get-ml-model-deepseek-r1/COPYRIGHT.md create mode 100644 script/get-ml-model-deepseek-r1/customize.py create mode 100644 script/get-ml-model-deepseek-r1/meta.yaml diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 55e79c0ae..86b93ac06 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -5,7 +5,6 @@ on: types: [published] workflow_dispatch: {} - jobs: build_wheels: if: github.repository_owner == 'mlcommons' diff --git a/script/get-ml-model-deepseek-r1/COPYRIGHT.md b/script/get-ml-model-deepseek-r1/COPYRIGHT.md new file mode 100644 index 000000000..d2ceead84 --- /dev/null +++ b/script/get-ml-model-deepseek-r1/COPYRIGHT.md @@ -0,0 +1,9 @@ +# Copyright Notice + +© 2025-2026 MLCommons. All Rights Reserved. + +This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: + +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. diff --git a/script/get-ml-model-deepseek-r1/customize.py b/script/get-ml-model-deepseek-r1/customize.py new file mode 100644 index 000000000..5f0d910fe --- /dev/null +++ b/script/get-ml-model-deepseek-r1/customize.py @@ -0,0 +1,31 @@ +from mlc import utils +import os +from utils import is_true + + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + if os_info['platform'] == "windows": + return {'return': 1, 'error': 'Script not supported in windows yet!'} + + if env.get('MLC_ML_MODEL_DEEPSEEK_R1_PATH', '') == '': + env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes" + elif not os.path.exists(env['MLC_ML_MODEL_DEEPSEEK_R1_PATH']): + return { + 'return': 1, 'error': f"Path {env['MLC_ML_MODEL_DEEPSEEK_R1_PATH']} does not exists!"} + + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + + if not env.get('MLC_DOWNLOAD_MODE', '') == "dry": + env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['MLC_ML_MODEL_DEEPSEEK_R1_PATH'] + + return {'return': 0} diff --git a/script/get-ml-model-deepseek-r1/meta.yaml b/script/get-ml-model-deepseek-r1/meta.yaml new file mode 100644 index 000000000..c055364ef --- /dev/null +++ b/script/get-ml-model-deepseek-r1/meta.yaml @@ -0,0 +1,58 @@ +alias: get-ml-model-deepseek-r1 +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +new_env_keys: +- MLC_ML_MODEL_DEEPSEEK_R1_PATH +- MLC_ML_MODEL_FILE_WITH_PATH +print_env_at_the_end: + MLC_ML_MODEL_DEEPSEEK_R1_PATH: DeepSeek-R1 checkpoint path +tags: +- get +- ml-model +- deepseek-r1 +- get-ml-model-deepseek-r1 +tests: + run_inputs: + - variations_list: + - r2-downloader,mlc,dry-run +uid: 2bcb61611b1f4af3 +variations: + dry-run: + env: + MLC_DOWNLOAD_MODE: dry + group: run-mode + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + mlc: + default: true + env: + MLC_DOWNLOAD_SRC: mlcommons + group: download-src + prehook_deps: + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_ML_MODEL_DEEPSEEK_R1_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_ML_MODEL_DEEPSEEK_R1_PATH + extra_cache_tags: ml,model,deepseek-r1 + force_cache: true + force_env_keys: + - MLC_OUTDIRNAME + names: + - dae-deepseek-r1 + tags: download-and-extract + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL + r2-downloader: + default: true + add_deps_recursive: + dae-deepseek-r1: + tags: _r2-downloader + group: download-tool + r2-downloader,mlc: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/deepseek-r1-0528.uri From 1f95493f0ac8bb169f59065212fb6d85249b2586 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 13 Oct 2025 15:34:38 +0000 Subject: [PATCH 29/55] [Automated Commit] Document script/get-ml-model-deepseek-r1/meta.yaml [skip ci] --- script/get-ml-model-deepseek-r1/README.md | 70 +++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 script/get-ml-model-deepseek-r1/README.md diff --git a/script/get-ml-model-deepseek-r1/README.md b/script/get-ml-model-deepseek-r1/README.md new file mode 100644 index 000000000..e3edc0e61 --- /dev/null +++ b/script/get-ml-model-deepseek-r1/README.md @@ -0,0 +1,70 @@ +# README for get-ml-model-deepseek-r1 +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,ml-model,deepseek-r1,get-ml-model-deepseek-r1 +``` + +No script specific inputs +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Download-src + +- `mlc` (default) + +### Download-tool + +- `r2-downloader` (default) + +### Run-mode + +- `dry-run` From cdbd9f5ff120d5336118c2adce3633f6619d51cb Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 13 Oct 2025 23:06:50 +0530 Subject: [PATCH 30/55] Support r2-downloader for dlrm model download script (#676) * support r2-downloader for dlrm * add dry-run variation --- script/get-ml-model-dlrm-terabyte/meta.yaml | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/script/get-ml-model-dlrm-terabyte/meta.yaml b/script/get-ml-model-dlrm-terabyte/meta.yaml index fed99aaa4..372960045 100644 --- a/script/get-ml-model-dlrm-terabyte/meta.yaml +++ b/script/get-ml-model-dlrm-terabyte/meta.yaml @@ -104,6 +104,9 @@ variations: env: MLC_PACKAGE_URL: mlc-inference:mlcommons-inference-wg-public/model_weights MLC_RCLONE_CONFIG_NAME: mlc-inference + pytorch,fp32,weight_sharded,r2-downloader: + env: + MLC_PACKAGE_URL: https://inference.mlcommons-storage.org/metadata/dlrm-v2-model-weights.uri pytorch,fp32,weight_sharded,wget: env: MLC_DAE_EXTRACT_DOWNLOADED: 'yes' @@ -115,6 +118,21 @@ variations: dae: tags: _rclone group: download-tool + r2-downloader: + ad: + dae: + tags: _r2-downloader + group: download-tool + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x weight_sharded: default: true env: @@ -125,3 +143,8 @@ variations: dae: tags: _wget group: download-tool +tests: + run_inputs: + - variations_list: + - pytorch,fp32,weight_sharded,r2-downloader,dry-run + - pytorch,fp32,weight_sharded,rclone,dry-run \ No newline at end of file From 91750ba2a93af1dfa4ccf45765d7de13974bd05b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 13 Oct 2025 17:37:20 +0000 Subject: [PATCH 31/55] [Automated Commit] Document script/get-ml-model-dlrm-terabyte/meta.yaml [skip ci] --- script/get-ml-model-dlrm-terabyte/README.md | 91 +++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 script/get-ml-model-dlrm-terabyte/README.md diff --git a/script/get-ml-model-dlrm-terabyte/README.md b/script/get-ml-model-dlrm-terabyte/README.md new file mode 100644 index 000000000..e4eb03be6 --- /dev/null +++ b/script/get-ml-model-dlrm-terabyte/README.md @@ -0,0 +1,91 @@ +# README for get-ml-model-dlrm-terabyte +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,ml-model,dlrm,raw,terabyte,criteo-terabyte,criteo,recommendation +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--dir` | | | `` | +| `--download_path` | Alias for dir | | `` | +| `--to` | Alias for dir | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Download-tool + +- `r2-downloader` +- `rclone` +- `wget` + +### Framework + +- `onnx` +- `pytorch` (default) + +### Precision + +- `fp32` (default) + +### Run-mode + +- `dry-run` + +### Type + +- `weight_sharded` (default) + +### Ungrouped + +- `debug` From 02209785b5da3f525ce06bd2c360c98822561abc Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 13 Oct 2025 23:08:27 +0530 Subject: [PATCH 32/55] Support r2-downloader for preprocessed criteo (#677) * support r2-downloader for preprocessed criteo * Support outdirname --- .../get-preprocessed-dataset-criteo/meta.yaml | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/script/get-preprocessed-dataset-criteo/meta.yaml b/script/get-preprocessed-dataset-criteo/meta.yaml index 7455121c2..56e1d98bd 100644 --- a/script/get-preprocessed-dataset-criteo/meta.yaml +++ b/script/get-preprocessed-dataset-criteo/meta.yaml @@ -141,22 +141,34 @@ variations: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_PREPROCESSED_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_PREPROCESSED_PATH MLC_EXTRACT_TO_FOLDER: criteo-preprocessed - MLC_RCLONE_CONFIG_NAME: mlc-inference - MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/dlrm_preprocessed extra_cache_tags: criteo,preprocessed,dataset force_cache: true + force_env_keys: + - MLC_OUTDIRNAME names: - - dae + - dae-criteo-preprocessed tags: download-and-extract update_tags_from_env_with_prefix: _url.: - MLC_DOWNLOAD_URL + multihot,mlc,rclone: + env: + MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/dlrm_preprocessed + MLC_RCLONE_CONFIG_NAME: mlc-inference + multihot,mlc,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/dlrm-v2-preprocessed-dataset.uri rclone: group: download-tool add_deps_recursive: - dae: + dae-criteo-preprocessed: tags: _rclone default: true + r2-downloader: + group: download-tool + add_deps_recursive: + dae-criteo-preprocessed: + tags: _r2-downloader dry-run: group: run-mode env: @@ -164,9 +176,17 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x preprocess: group: src validation: add_deps: original-dataset: tags: _validation +tests: + run_inputs: + - variations_list: + - r2-downloader,mlc,dry-run + - rclone,mlc,dry-run From 018c1dd94839f95e14c4634b533f943c194a4722 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 13 Oct 2025 17:38:57 +0000 Subject: [PATCH 33/55] [Automated Commit] Document script/get-preprocessed-dataset-criteo/meta.yaml [skip ci] --- .../get-preprocessed-dataset-criteo/README.md | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 script/get-preprocessed-dataset-criteo/README.md diff --git a/script/get-preprocessed-dataset-criteo/README.md b/script/get-preprocessed-dataset-criteo/README.md new file mode 100644 index 000000000..8141649ac --- /dev/null +++ b/script/get-preprocessed-dataset-criteo/README.md @@ -0,0 +1,90 @@ +# README for get-preprocessed-dataset-criteo +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,dataset,criteo,recommendation,dlrm,preprocessed +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--dir` | | | `` | +| `--output_dir` | | | `` | +| `--threads` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Download-tool + +- `r2-downloader` +- `rclone` (default) + +### Run-mode + +- `dry-run` + +### Src + +- `mlc` (default) +- `preprocess` + +### Type + +- `multihot` (default) + +### Ungrouped + +- `1` +- `50` +- `fake` +- `full` +- `validation` From 34c5f7f49b23e44fecfcd3f8a180c7d6d58cf236 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 14 Oct 2025 23:20:14 +0530 Subject: [PATCH 34/55] Support Mixtral model and openorca download through r2 downloader (#678) * Support mixtral model download through r2 * Remove r2 from being default * add dry run variation --- script/get-ml-model-mixtral/customize.py | 11 +- script/get-ml-model-mixtral/meta.yaml | 43 +++++++ .../customize.py | 120 +++++++++--------- .../meta.yaml | 40 +++++- 4 files changed, 147 insertions(+), 67 deletions(-) diff --git a/script/get-ml-model-mixtral/customize.py b/script/get-ml-model-mixtral/customize.py index d33656b33..20adf59f4 100644 --- a/script/get-ml-model-mixtral/customize.py +++ b/script/get-ml-model-mixtral/customize.py @@ -19,10 +19,11 @@ def postprocess(i): env = i['env'] - if env.get('MIXTRAL_CHECKPOINT_PATH', '') == '': - env['MIXTRAL_CHECKPOINT_PATH'] = env['MLC_ML_MODEL_PATH'] - else: - env['MLC_ML_MODEL_PATH'] = env['MIXTRAL_CHECKPOINT_PATH'] - env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_ML_MODEL_PATH'] + if not env.get('MLC_DOWNLOAD_MODE', '') == "dry": + if env.get('MIXTRAL_CHECKPOINT_PATH', '') == '': + env['MIXTRAL_CHECKPOINT_PATH'] = env['MLC_ML_MODEL_PATH'] + else: + env['MLC_ML_MODEL_PATH'] = env['MIXTRAL_CHECKPOINT_PATH'] + env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_ML_MODEL_PATH'] return {'return': 0} diff --git a/script/get-ml-model-mixtral/meta.yaml b/script/get-ml-model-mixtral/meta.yaml index 48ded49db..804f33e4f 100644 --- a/script/get-ml-model-mixtral/meta.yaml +++ b/script/get-ml-model-mixtral/meta.yaml @@ -17,6 +17,9 @@ prehook_deps: - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' + skip_if_env: + MLC_DOWNLOAD_TOOL: + - r2-downloader env: {} extra_cache_tags: mixtral force_env_keys: @@ -57,6 +60,42 @@ variations: MLC_ML_MODEL_STARTING_WEIGHTS_FILENAME: https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1 MLC_MODEL_ZOO_ENV_KEY: MIXTRAL group: huggingface-stub + mlc: + prehook_deps: + - env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MIXTRAL_CHECKPOINT_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MIXTRAL_CHECKPOINT_PATH + extra_cache_tags: ml-model,mixtral,raw,ml-model-mixtral + force_cache: true + names: + - dae-mixtral-model + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_PACKAGE_URL + mlc,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/mixtral-8x7b-model-checkpoint.uri + MLC_ML_MODEL_STARTING_WEIGHTS_FILENAME: "" + r2-downloader: + adr: + dae-mixtral-model: + tags: _r2-downloader + env: + MLC_DOWNLOAD_TOOL: r2-downloader + group: download-tool + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x pytorch: default: true env: @@ -69,3 +108,7 @@ variations: env: MLC_MODEL_ZOO_ENV_KEY: MIXTRAL group: huggingface-stub +tests: + run_inputs: + - variations_list: + - r2-downloader,mlc,dry-run diff --git a/script/get-preprocessed-dataset-openorca/customize.py b/script/get-preprocessed-dataset-openorca/customize.py index 4d6af7503..80b61b74d 100644 --- a/script/get-preprocessed-dataset-openorca/customize.py +++ b/script/get-preprocessed-dataset-openorca/customize.py @@ -8,38 +8,39 @@ def preprocess(i): env = i['env'] - if is_true(str(env.get('MLC_DATASET_PREPROCESSED_BY_MLC', ''))): - run_dir = env['MLC_OPENORCA_PREPROCESSED_ROOT'] - run_cmd = '' - env['MLC_DATASET_CALIBRATION_PATH'] = os.path.join( - env['MLC_OPENORCA_PREPROCESSED_ROOT'], - "open_orca_gpt4_tokenized_llama.calibration_1000.pkl.gz") - run_cmd = f"gzip -dkf {env['MLC_DATASET_CALIBRATION_PATH']}" - env['MLC_DATASET_PREPROCESSED_PATH'] = os.path.join( - env['MLC_OPENORCA_PREPROCESSED_ROOT'], - "open_orca_gpt4_tokenized_llama.sampled_24576.pkl.gz") - run_cmd += f" && gzip -dkf {env['MLC_DATASET_PREPROCESSED_PATH']}" - else: - inference_src = env['MLC_MLPERF_INFERENCE_SOURCE'] - run_dir = os.path.join(inference_src, 'language', 'llama2-70b') - model_dir = env['MLC_ML_MODEL_PATH'] - if is_true(env.get('MLC_DATASET_CALIBRATION', '')): - return {'return': 1, 'error': 'No raw preprocessing information is available for openorca calibration. Please use _mlcommons variation to use the MLCommons shared calibration dataset'} - else: + if not env.get('MLC_DOWNLOAD_MODE', '') == "dry": + if is_true(str(env.get('MLC_DATASET_PREPROCESSED_BY_MLC', ''))): + run_dir = env['MLC_OPENORCA_PREPROCESSED_ROOT'] + run_cmd = '' + env['MLC_DATASET_CALIBRATION_PATH'] = os.path.join( + env['MLC_OPENORCA_PREPROCESSED_ROOT'], + "open_orca_gpt4_tokenized_llama.calibration_1000.pkl.gz") + run_cmd = f"gzip -dkf {env['MLC_DATASET_CALIBRATION_PATH']}" env['MLC_DATASET_PREPROCESSED_PATH'] = os.path.join( - os.path.join( - os.getcwd(), - "processed-openorca", - 'open_orca_gpt4_tokenized_llama.sampled_' + - env['MLC_DATASET_SIZE'] + - '.pkl')) - run_cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + ' processorca.py --dataset_pq_path=' + env['MLC_DATASET_OPENORCA_PARQUET'] + ' --model_dir=' + model_dir + \ - ' --seqlen_limit=2048 --export_dir=' + \ - os.path.join(os.getcwd(), "processed-openorca") + \ - ' --num_total_samples=' + env['MLC_DATASET_SIZE'] + env['MLC_OPENORCA_PREPROCESSED_ROOT'], + "open_orca_gpt4_tokenized_llama.sampled_24576.pkl.gz") + run_cmd += f" && gzip -dkf {env['MLC_DATASET_PREPROCESSED_PATH']}" + else: + inference_src = env['MLC_MLPERF_INFERENCE_SOURCE'] + run_dir = os.path.join(inference_src, 'language', 'llama2-70b') + model_dir = env['MLC_ML_MODEL_PATH'] + if is_true(env.get('MLC_DATASET_CALIBRATION', '')): + return {'return': 1, 'error': 'No raw preprocessing information is available for openorca calibration. Please use _mlcommons variation to use the MLCommons shared calibration dataset'} + else: + env['MLC_DATASET_PREPROCESSED_PATH'] = os.path.join( + os.path.join( + os.getcwd(), + "processed-openorca", + 'open_orca_gpt4_tokenized_llama.sampled_' + + env['MLC_DATASET_SIZE'] + + '.pkl')) + run_cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + ' processorca.py --dataset_pq_path=' + env['MLC_DATASET_OPENORCA_PARQUET'] + ' --model_dir=' + model_dir + \ + ' --seqlen_limit=2048 --export_dir=' + \ + os.path.join(os.getcwd(), "processed-openorca") + \ + ' --num_total_samples=' + env['MLC_DATASET_SIZE'] - env['MLC_RUN_DIR'] = run_dir - env['MLC_RUN_CMD'] = run_cmd + env['MLC_RUN_DIR'] = run_dir + env['MLC_RUN_CMD'] = run_cmd return {'return': 0} @@ -47,36 +48,37 @@ def preprocess(i): def postprocess(i): env = i['env'] - if is_true(str(env.get('MLC_DATASET_PREPROCESSED_BY_MLC', ''))): - env['PREPROCESSED_DATA_DIR'] = os.path.dirname( - env['MLC_OPENORCA_PREPROCESSED_ROOT']) - if is_true(env.get('MLC_DATASET_CALIBRATION', '')): - if env.get('MLC_TMP_DATASET_PREPROCESS_STEP_PROVIDER', - '') == "nvidia": - env['MLC_NVIDIA_PREPROCESSED_CALIBRATION_DATASET_PATH'] = os.path.join( - env['MLC_OPENORCA_PREPROCESSED_ROOT'], - "preprocessed_data", - "mlperf_llama2_openorca_calibration_1k") + if not env.get('MLC_DOWNLOAD_MODE', '') == "dry": + if is_true(str(env.get('MLC_DATASET_PREPROCESSED_BY_MLC', ''))): + env['PREPROCESSED_DATA_DIR'] = os.path.dirname( + env['MLC_OPENORCA_PREPROCESSED_ROOT']) + if is_true(env.get('MLC_DATASET_CALIBRATION', '')): + if env.get('MLC_TMP_DATASET_PREPROCESS_STEP_PROVIDER', + '') == "nvidia": + env['MLC_NVIDIA_PREPROCESSED_CALIBRATION_DATASET_PATH'] = os.path.join( + env['MLC_OPENORCA_PREPROCESSED_ROOT'], + "preprocessed_data", + "mlperf_llama2_openorca_calibration_1k") + else: + env['MLC_DATASET_CALIBRATION_PATH'] = os.path.join( + env['MLC_OPENORCA_PREPROCESSED_ROOT'], + "open_orca_gpt4_tokenized_llama.calibration_1000.pkl") + env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_DATASET_CALIBRATION_PATH'] + env['MLC_DATASET_OPENORCA_CALIBRATION_PATH'] = env['MLC_DATASET_CALIBRATION_PATH'] else: - env['MLC_DATASET_CALIBRATION_PATH'] = os.path.join( + env['MLC_DATASET_PREPROCESSED_PATH'] = os.path.join( env['MLC_OPENORCA_PREPROCESSED_ROOT'], - "open_orca_gpt4_tokenized_llama.calibration_1000.pkl") - env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_DATASET_CALIBRATION_PATH'] - env['MLC_DATASET_OPENORCA_CALIBRATION_PATH'] = env['MLC_DATASET_CALIBRATION_PATH'] - else: - env['MLC_DATASET_PREPROCESSED_PATH'] = os.path.join( - env['MLC_OPENORCA_PREPROCESSED_ROOT'], - "open_orca_gpt4_tokenized_llama.sampled_24576.pkl") - if env.get('MLC_TMP_DATASET_PREPROCESS_STEP_PROVIDER', - '') == "nvidia": - env['MLC_DATASET_OPENORCA_NVIDIA_PREPROCESSED_PATH'] = os.path.join( - env['MLC_OPENORCA_PREPROCESSED_ROOT'], - "preprocessed_data") - env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_DATASET_OPENORCA_NVIDIA_PREPROCESSED_PATH'] - else: - env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_DATASET_PREPROCESSED_PATH'] - # The openorca mlc preprocessed dataset is used in nvidia - # implementation for checking accuracy - env['MLC_DATASET_OPENORCA_PREPROCESSED_PATH'] = env['MLC_DATASET_PREPROCESSED_PATH'] + "open_orca_gpt4_tokenized_llama.sampled_24576.pkl") + if env.get('MLC_TMP_DATASET_PREPROCESS_STEP_PROVIDER', + '') == "nvidia": + env['MLC_DATASET_OPENORCA_NVIDIA_PREPROCESSED_PATH'] = os.path.join( + env['MLC_OPENORCA_PREPROCESSED_ROOT'], + "preprocessed_data") + env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_DATASET_OPENORCA_NVIDIA_PREPROCESSED_PATH'] + else: + env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_DATASET_PREPROCESSED_PATH'] + # The openorca mlc preprocessed dataset is used in nvidia + # implementation for checking accuracy + env['MLC_DATASET_OPENORCA_PREPROCESSED_PATH'] = env['MLC_DATASET_PREPROCESSED_PATH'] return {'return': 0} diff --git a/script/get-preprocessed-dataset-openorca/meta.yaml b/script/get-preprocessed-dataset-openorca/meta.yaml index 258accbeb..d3425e085 100644 --- a/script/get-preprocessed-dataset-openorca/meta.yaml +++ b/script/get-preprocessed-dataset-openorca/meta.yaml @@ -105,24 +105,53 @@ variations: group: size mlc: alias: mlcommons + r2-downloader: + add_deps_recursive: + dae-openorca: + tags: _r2-downloader + group: download-tool + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + rclone: + add_deps_recursive: + dae-openorca: + tags: _rclone + default: true + group: download-tool + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run mlcommons: deps: - env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_OPENORCA_PREPROCESSED_ROOT MLC_EXTRACT_FINAL_ENV_NAME: MLC_OPENORCA_PREPROCESSED_ROOT MLC_EXTRACT_TO_FOLDER: openorca-preprocessed - MLC_RCLONE_CONFIG_NAME: mlc-inference extra_cache_tags: openorca,preprocessed,dataset force_cache: true names: - dae-openorca - tags: download-and-extract,_rclone + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME update_tags_from_env_with_prefix: _url.: - - MLC_RCLONE_URL + - MLC_DOWNLOAD_URL env: MLC_DATASET_PREPROCESSED_BY_MLC: 'yes' MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/open_orca + r2-downloader,mlc: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/llama-2-70b-open-orca-dataset.uri + rclone,mlc: + env: + MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/open_orca + MLC_RCLONE_CONFIG_NAME: mlc-inference size.#: ad: dataset-original: @@ -147,3 +176,8 @@ variations: nvidia,validation: new_env_keys: - MLC_DATASET_OPENORCA_NVIDIA_PREPROCESSED_PATH +tests: + run_inputs: + - variations_list: + - r2-downloader,mlc,dry-run + - rclone,mlc,dry-run From 6ac230604c41831112fd7b93f6d78269976e8cf4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 14 Oct 2025 17:50:44 +0000 Subject: [PATCH 35/55] [Automated Commit] Document script/get-ml-model-mixtral/meta.yaml [skip ci] --- script/get-ml-model-mixtral/README.md | 88 +++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 script/get-ml-model-mixtral/README.md diff --git a/script/get-ml-model-mixtral/README.md b/script/get-ml-model-mixtral/README.md new file mode 100644 index 000000000..1649edfc4 --- /dev/null +++ b/script/get-ml-model-mixtral/README.md @@ -0,0 +1,88 @@ +# README for get-ml-model-mixtral +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,raw,ml-model,language-processing,mixtral,mixtral-8x7b +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--checkpoint` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Download-tool + +- `r2-downloader` + +### Framework + +- `pytorch` (default) + +### Huggingface-stub + +- `mistralai/Mixtral-8x7B-Instruct-v0.1` (default) +- `stub.#` _(# can be substituted dynamically)_ + +### Precision + +- `fp32` (default) + +### Run-mode + +- `dry-run` + +### Ungrouped + +- `batch_size.#` _(# can be substituted dynamically)_ +- `mlc` From 1178a8b55d7ad4401bfbf10258e6767bc18e7543 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 14 Oct 2025 23:20:50 +0530 Subject: [PATCH 36/55] Support r2 for mixtral dataset (#679) --- .../customize.py | 13 ++++-- .../meta.yaml | 46 +++++++++++++++++-- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/script/get-dataset-mlperf-inference-mixtral/customize.py b/script/get-dataset-mlperf-inference-mixtral/customize.py index 5ecfbafdf..ef71a071a 100644 --- a/script/get-dataset-mlperf-inference-mixtral/customize.py +++ b/script/get-dataset-mlperf-inference-mixtral/customize.py @@ -19,9 +19,16 @@ def preprocess(i): def postprocess(i): env = i['env'] - env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = env['MLC_DATASET_PREPROCESSED_PATH'] + if not env.get('MLC_DOWNLOAD_MODE', '') == "dry": + env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = env['MLC_DATASET_PREPROCESSED_PATH'] - if is_true(env.get('MLC_DATASET_MIXTRAL_GENERATE_TEST_DATA', '')): - env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = env['MLC_DATASET_MIXTRAL_TEST_DATA_GENERATED_PATH'] + if env.get('MLC_DOWNLOAD_TOOL', '') == "r2-downloader": + if env.get('MLC_DATASET_TYPE', '') == 'validation': + env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = os.path.join(env['MLC_DATASET_PREPROCESSED_PATH'], '09292024_mixtral_15k_mintoken2_v1.pkl') + elif env.get('MLC_DATASET_TYPE', '') == 'calibration': + env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = os.path.join(env['MLC_DATASET_PREPROCESSED_PATH'], '2024.06.06_mixtral_15k_calibration_v4.pkl') + + if is_true(env.get('MLC_DATASET_MIXTRAL_GENERATE_TEST_DATA', '')): + env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = env['MLC_DATASET_MIXTRAL_TEST_DATA_GENERATED_PATH'] return {'return': 0} diff --git a/script/get-dataset-mlperf-inference-mixtral/meta.yaml b/script/get-dataset-mlperf-inference-mixtral/meta.yaml index 744b5690d..442510ebd 100644 --- a/script/get-dataset-mlperf-inference-mixtral/meta.yaml +++ b/script/get-dataset-mlperf-inference-mixtral/meta.yaml @@ -17,7 +17,7 @@ prehook_deps: _url.: - MLC_PACKAGE_URL names: - - download-file + - download-file-mixtral-dataset tags: - get - dataset-mixtral @@ -28,15 +28,22 @@ variations: group: download-tool default: true adr: - download-file: + download-file-mixtral-dataset: tags: _rclone env: MLC_DOWNLOAD_TOOL: rclone MLC_RCLONE_COPY_USING: copyurl + r2-downloader: + group: download-tool + adr: + download-file-mixtral-dataset: + tags: _r2-downloader + env: + MLC_DOWNLOAD_TOOL: r2-downloader wget: group: download-tool adr: - download-file: + download-file-mixtral-dataset: tags: _wget env: MLC_DOWNLOAD_TOOL: wget @@ -44,19 +51,31 @@ variations: default: true group: dataset-type adr: - download-file: + download-file-mixtral-dataset: extra_cache_tags: mixtral,get-mixtral-dataset,validation env: ML_MODEL_STARTING_WEIGHTS_FILENAME: https://github.com/mlcommons/inference/tree/master/language/mixtral-8x7b#preprocessed + MLC_DATASET_TYPE: validation + validation,rclone: + env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/mixtral_8x7b/09292024_mixtral_15k_mintoken2_v1.pkl + validation,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/mixtral-8x7b-validation-dataset.uri calibration: group: dataset-type adr: - download-file: + download-file-mixtral-dataset: extra_cache_tags: mixtral,get-mixtral-dataset,calibration env: ML_MODEL_STARTING_WEIGHTS_FILENAME: https://github.com/mlcommons/inference/tree/master/language/mixtral-8x7b#calibration-dataset + MLC_DATASET_TYPE: calibration + calibration,rclone: + env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/mixtral_8x7b%2F2024.06.06_mixtral_15k_calibration_v4.pkl + calibration,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/mixtral-8x7b-calibration-dataset.uri mlcommons-storage: env: MLC_DOWNLOAD_CHECKSUM: 78823c13e0e73e518872105c4b09628b @@ -72,3 +91,20 @@ variations: env: MLC_DATASET_MIXTRAL_GENERATE_TEST_DATA: 'yes' MLC_DATASET_MIXTRAL_TEST_DATA_SIZE: '#' + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run +tests: + run_inputs: + - variations_list: + - r2-downloader,calibration,dry-run + - rclone,calibration,dry-run + - r2-downloader,validation,dry-run + - rclone,validation,dry-run From 31b48fd5a46b867a6da25456682b047f5f90f76c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 14 Oct 2025 17:51:01 +0000 Subject: [PATCH 37/55] [Automated Commit] Format Codebase [skip ci] --- script/get-dataset-mlperf-inference-mixtral/customize.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/script/get-dataset-mlperf-inference-mixtral/customize.py b/script/get-dataset-mlperf-inference-mixtral/customize.py index ef71a071a..e233ebf50 100644 --- a/script/get-dataset-mlperf-inference-mixtral/customize.py +++ b/script/get-dataset-mlperf-inference-mixtral/customize.py @@ -24,9 +24,11 @@ def postprocess(i): if env.get('MLC_DOWNLOAD_TOOL', '') == "r2-downloader": if env.get('MLC_DATASET_TYPE', '') == 'validation': - env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = os.path.join(env['MLC_DATASET_PREPROCESSED_PATH'], '09292024_mixtral_15k_mintoken2_v1.pkl') + env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = os.path.join( + env['MLC_DATASET_PREPROCESSED_PATH'], '09292024_mixtral_15k_mintoken2_v1.pkl') elif env.get('MLC_DATASET_TYPE', '') == 'calibration': - env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = os.path.join(env['MLC_DATASET_PREPROCESSED_PATH'], '2024.06.06_mixtral_15k_calibration_v4.pkl') + env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = os.path.join( + env['MLC_DATASET_PREPROCESSED_PATH'], '2024.06.06_mixtral_15k_calibration_v4.pkl') if is_true(env.get('MLC_DATASET_MIXTRAL_GENERATE_TEST_DATA', '')): env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = env['MLC_DATASET_MIXTRAL_TEST_DATA_GENERATED_PATH'] From 8bc886ce71e6d9659b35d32afcc7524f620d5fa5 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 14 Oct 2025 23:26:13 +0530 Subject: [PATCH 38/55] Add support for adding r2-downoader for GPT-j (#680) * initial commit for supporting r2 downloader in gptj * fix download-tool group issue --- script/get-dataset-cnndm/meta.yaml | 4 ++ script/get-ml-model-gptj/customize.py | 65 ++++++++++++++------------- script/get-ml-model-gptj/meta.yaml | 33 +++++++++++--- 3 files changed, 66 insertions(+), 36 deletions(-) diff --git a/script/get-dataset-cnndm/meta.yaml b/script/get-dataset-cnndm/meta.yaml index 8214fe123..05de6ac0e 100644 --- a/script/get-dataset-cnndm/meta.yaml +++ b/script/get-dataset-cnndm/meta.yaml @@ -35,6 +35,10 @@ deps: skip_if_env: MLC_TMP_ML_MODEL: - llama3_1-8b +- tags: get,generic-python-lib,_package.transformers + skip_if_env: + MLC_TMP_ML_MODEL: + - llama3_1-8b - tags: get,generic-python-lib,_numpy skip_if_env: MLC_TMP_ML_MODEL: diff --git a/script/get-ml-model-gptj/customize.py b/script/get-ml-model-gptj/customize.py index 8d0ca4e9c..6a88b7fe1 100644 --- a/script/get-ml-model-gptj/customize.py +++ b/script/get-ml-model-gptj/customize.py @@ -67,39 +67,42 @@ def postprocess(i): env = i['env'] - if os.path.exists(os.path.join( - env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final")): - env['GPTJ_CHECKPOINT_PATH'] = os.path.join( - env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final") - - is_saxml = env.get('MLC_TMP_MODEL_SAXML', '') - if is_saxml == "fp32": - if os.path.exists("pax_gptj_checkpoint"): - env['GPTJ_SAXML_CHECKPOINT_PATH'] = os.path.join( - os.getcwd(), "pax_gptj_checkpoint") - env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_SAXML_CHECKPOINT_PATH'] - else: - return {'return': 1, 'error': 'pax_gptj_checkpoint generation failed'} + if not env.get('MLC_DOWNLOAD_MODE', '') == "dry": + if os.path.exists(os.path.join( + env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final")): + env['GPTJ_CHECKPOINT_PATH'] = os.path.join( + env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final") + + is_saxml = env.get('MLC_TMP_MODEL_SAXML', '') + if is_saxml == "fp32": + if os.path.exists("pax_gptj_checkpoint"): + env['GPTJ_SAXML_CHECKPOINT_PATH'] = os.path.join( + os.getcwd(), "pax_gptj_checkpoint") + env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_SAXML_CHECKPOINT_PATH'] + else: + return {'return': 1, + 'error': 'pax_gptj_checkpoint generation failed'} - elif is_saxml == "int8": - if os.path.exists("int8_ckpt"): - env['GPTJ_SAXML_INT8_CHECKPOINT_PATH'] = os.path.join( - os.getcwd(), "int8_ckpt") - env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_SAXML_INT8_CHECKPOINT_PATH'] + elif is_saxml == "int8": + if os.path.exists("int8_ckpt"): + env['GPTJ_SAXML_INT8_CHECKPOINT_PATH'] = os.path.join( + os.getcwd(), "int8_ckpt") + env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_SAXML_INT8_CHECKPOINT_PATH'] + else: + return {'return': 1, + 'error': 'pax_gptj_checkpoint generation failed'} + elif env.get('MLC_TMP_ML_MODEL_PROVIDER', '') == 'nvidia': + env['MLC_ML_MODEL_FILE_WITH_PATH'] = os.path.join( + env['MLC_NVIDIA_MLPERF_SCRATCH_PATH'], + 'models', + 'GPTJ-6B', + 'fp8-quantized-ammo', + 'GPTJ-FP8-quantized') else: - return {'return': 1, 'error': 'pax_gptj_checkpoint generation failed'} - elif env.get('MLC_TMP_ML_MODEL_PROVIDER', '') == 'nvidia': - env['MLC_ML_MODEL_FILE_WITH_PATH'] = os.path.join( - env['MLC_NVIDIA_MLPERF_SCRATCH_PATH'], - 'models', - 'GPTJ-6B', - 'fp8-quantized-ammo', - 'GPTJ-FP8-quantized') - else: - env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_CHECKPOINT_PATH'] + env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_CHECKPOINT_PATH'] - env['MLC_ML_MODEL_FILE'] = os.path.basename( - env['MLC_ML_MODEL_FILE_WITH_PATH']) - env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_ML_MODEL_FILE_WITH_PATH'] + env['MLC_ML_MODEL_FILE'] = os.path.basename( + env['MLC_ML_MODEL_FILE_WITH_PATH']) + env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_ML_MODEL_FILE_WITH_PATH'] return {'return': 0} diff --git a/script/get-ml-model-gptj/meta.yaml b/script/get-ml-model-gptj/meta.yaml index 3c7e86cb8..f9d53576a 100644 --- a/script/get-ml-model-gptj/meta.yaml +++ b/script/get-ml-model-gptj/meta.yaml @@ -37,7 +37,8 @@ prehook_deps: tests: run_inputs: - variations_list: - - fp32 + - fp32,pytorch,r2-downloader,dry-run + - fp32,pytorch,rclone,dry-run print_env_at_the_end: @@ -102,10 +103,15 @@ variations: env: MLC_DOWNLOAD_CHECKSUM_NOT_USED: e677e28aaf03da84584bb3073b7ee315 MLC_PACKAGE_URL: https://cloud.mlcommons.org/index.php/s/QAZ2oM94MkFtbQx/download - MLC_RCLONE_CONFIG_NAME: mlc-inference - MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/gpt-j MLC_UNZIP: 'yes' required_disk_space: 22700 + pytorch,fp32,rclone: + env: + MLC_PACKAGE_URL: mlc-inference:mlcommons-inference-wg-public/gpt-j + MLC_RCLONE_CONFIG_NAME: mlc-inference + pytorch,fp32,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/gpt-j-model-checkpoint.uri pytorch,fp32,wget: add_deps_recursive: dae: @@ -183,11 +189,28 @@ variations: add_deps_recursive: dae: tags: _rclone - default: true env: MLC_DOWNLOAD_FILENAME: checkpoint - MLC_DOWNLOAD_URL: <<>> + MLC_DOWNLOAD_URL: <<>> group: download-tool + r2-downloader: + group: download-tool + default: true + add_deps_recursive: + dae: + tags: _r2-downloader + env: + MLC_DOWNLOAD_FILENAME: checkpoint + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run saxml: group: framework saxml,fp32: From 1a8489edd59a070ab9a8426fe71d6aed8ff42e2a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 14 Oct 2025 17:56:39 +0000 Subject: [PATCH 39/55] [Automated Commit] Document script/get-ml-model-gptj/meta.yaml [skip ci] --- script/get-ml-model-gptj/README.md | 97 ++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 script/get-ml-model-gptj/README.md diff --git a/script/get-ml-model-gptj/README.md b/script/get-ml-model-gptj/README.md new file mode 100644 index 000000000..76eaa9c8b --- /dev/null +++ b/script/get-ml-model-gptj/README.md @@ -0,0 +1,97 @@ +# README for get-ml-model-gptj +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,raw,ml-model,gptj,gpt-j,large-language-model +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--checkpoint` | | | `` | +| `--download_path` | | | `` | +| `--to` | Alias for download_path | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Download-tool + +- `r2-downloader` (default) +- `rclone` +- `wget` + +### Framework + +- `pytorch` (default) +- `saxml` + +### Model-provider + +- `intel` +- `mlcommons` (default) +- `nvidia` + +### Precision + +- `fp32` +- `fp8` +- `int4` +- `int8` +- `uint8` + +### Run-mode + +- `dry-run` + +### Ungrouped + +- `batch_size.#` _(# can be substituted dynamically)_ From 13a8b5fd1bb05ac9c694a5096e3b7ec7518c41e7 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 16 Oct 2025 13:07:35 +0530 Subject: [PATCH 40/55] Fix for openorca nvidia preprocessing (#686) --- script/get-preprocessed-dataset-openorca/meta.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/script/get-preprocessed-dataset-openorca/meta.yaml b/script/get-preprocessed-dataset-openorca/meta.yaml index d3425e085..e27947e1a 100644 --- a/script/get-preprocessed-dataset-openorca/meta.yaml +++ b/script/get-preprocessed-dataset-openorca/meta.yaml @@ -110,6 +110,7 @@ variations: dae-openorca: tags: _r2-downloader group: download-tool + default: true dry-run: group: run-mode env: @@ -118,7 +119,6 @@ variations: add_deps_recursive: dae-openorca: tags: _rclone - default: true group: download-tool dry-run,r2-downloader: env: @@ -127,6 +127,7 @@ variations: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run mlcommons: + default: true deps: - env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_OPENORCA_PREPROCESSED_ROOT @@ -145,10 +146,10 @@ variations: env: MLC_DATASET_PREPROCESSED_BY_MLC: 'yes' MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/open_orca - r2-downloader,mlc: + r2-downloader,mlcommons: env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/llama-2-70b-open-orca-dataset.uri - rclone,mlc: + rclone,mlcommons: env: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/open_orca MLC_RCLONE_CONFIG_NAME: mlc-inference From 940f9f3e0661a6cf49cc4172ed66979260044267 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Oct 2025 07:38:05 +0000 Subject: [PATCH 41/55] [Automated Commit] Document script/get-preprocessed-dataset-openorca/meta.yaml [skip ci] --- script/get-preprocessed-dataset-openorca/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/script/get-preprocessed-dataset-openorca/README.md b/script/get-preprocessed-dataset-openorca/README.md index 5899d8c1f..c1f52bc0e 100644 --- a/script/get-preprocessed-dataset-openorca/README.md +++ b/script/get-preprocessed-dataset-openorca/README.md @@ -62,10 +62,19 @@ No script specific inputs - `calibration` (base: mlcommons) - `validation` (default) +### Download-tool + +- `r2-downloader` (default) +- `rclone` + ### Preprocess-step-provider - `nvidia` +### Run-mode + +- `dry-run` + ### Size - `60` From 3e0e430258448e119b7ef221fdc7742cd2f6af7f Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 16 Oct 2025 19:51:55 +0530 Subject: [PATCH 42/55] support r2-downloader for waymo calibration dataset download (#681) * support r2-downloader for waymo calibration dataset download * Update filename env variable * add needs pat for calibration dataset waymo --- .../get-dataset-waymo-calibration/meta.yaml | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/script/get-dataset-waymo-calibration/meta.yaml b/script/get-dataset-waymo-calibration/meta.yaml index 96b916764..f952c7e1b 100644 --- a/script/get-dataset-waymo-calibration/meta.yaml +++ b/script/get-dataset-waymo-calibration/meta.yaml @@ -24,26 +24,16 @@ variations: MLC_DOWNLOAD_SRC: mlcommons group: download-src prehook_deps: - - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - true - tags: get,rclone - - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - true - force_cache: true - tags: get,rclone-config,_waymo - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_WAYMO_CALIBRATION_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_WAYMO_CALIBRATION_PATH - MLC_DOWNLOAD_URL: mlc_waymo:waymo_preprocessed_dataset/kitti_format/testing extra_cache_tags: waymo,dataset force_cache: true names: - - dae + - dae-waymo-calib tags: download-and-extract force_env_keys: - MLC_OUTDIRNAME @@ -53,9 +43,31 @@ variations: rclone: group: download-tool add_deps_recursive: - dae: + dae-waymo-calib: tags: _rclone + prehook_deps: + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - true + tags: get,rclone + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - true + force_cache: true + tags: get,rclone-config,_waymo + rclone,mlc: + env: + MLC_DOWNLOAD_URL: mlc_waymo:waymo_preprocessed_dataset/kitti_format/testing + r2-downloader: + group: download-tool + add_deps_recursive: + dae-waymo-calib: + tags: _r2-downloader default: true + r2-downloader,mlc: + env: + MLC_DOWNLOAD_URL: https://waymo.mlcommons-storage.org/metadata/calibration-dataset.uri + MLC_DOWNLOAD_FILENAME: calibration-dataset dry-run: group: run-mode env: @@ -63,4 +75,13 @@ variations: MLC_BYPASS_RCLONE_AUTH: True dry-run,rclone: env: - MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run \ No newline at end of file + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run +tests: + needs_pat: true + run_inputs: + - variations_list: + - r2-downloader,mlc,dry-run + - rclone,mlc,dry-run From b2327e54527b3618f883268bd7c2d90d055f0597 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Oct 2025 14:22:29 +0000 Subject: [PATCH 43/55] [Automated Commit] Document script/get-dataset-waymo-calibration/meta.yaml [skip ci] --- .../get-dataset-waymo-calibration/README.md | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 script/get-dataset-waymo-calibration/README.md diff --git a/script/get-dataset-waymo-calibration/README.md b/script/get-dataset-waymo-calibration/README.md new file mode 100644 index 000000000..19a18b39e --- /dev/null +++ b/script/get-dataset-waymo-calibration/README.md @@ -0,0 +1,79 @@ +# README for get-dataset-waymo-calibration +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,waymo,dataset,calibration +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--waymo_calibration_path` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Dataset-format + +- `kitti_format` (default) + +### Download-src + +- `mlc` (default) + +### Download-tool + +- `r2-downloader` (default) +- `rclone` + +### Run-mode + +- `dry-run` From b775acf48b4343f7465c665b919257a6edfc7c1c Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 16 Oct 2025 19:54:19 +0530 Subject: [PATCH 44/55] Add tests for get-dataset-waymo (#682) --- script/get-dataset-waymo/meta.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/script/get-dataset-waymo/meta.yaml b/script/get-dataset-waymo/meta.yaml index e90315017..953468644 100644 --- a/script/get-dataset-waymo/meta.yaml +++ b/script/get-dataset-waymo/meta.yaml @@ -76,3 +76,10 @@ variations: r2-downloader,mlc: env: MLC_DOWNLOAD_URL: https://waymo.mlcommons-storage.org/metadata/dataset.uri + MLC_DOWNLOAD_FILENAME: dataset +tests: + needs_pat: true + run_inputs: + - variations_list: + - r2-downloader,mlc,dry-run + - rclone,mlc,dry-run From 411a471c2da7ae0ecabd65cd09e0d176bb1b768a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Oct 2025 14:24:49 +0000 Subject: [PATCH 45/55] [Automated Commit] Document script/get-dataset-waymo/meta.yaml [skip ci] --- script/get-dataset-waymo/README.md | 79 ++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 script/get-dataset-waymo/README.md diff --git a/script/get-dataset-waymo/README.md b/script/get-dataset-waymo/README.md new file mode 100644 index 000000000..a9b89bde2 --- /dev/null +++ b/script/get-dataset-waymo/README.md @@ -0,0 +1,79 @@ +# README for get-dataset-waymo +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,dataset,waymo +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--waymo_path` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Dataset-format + +- `kitti_format` (default) + +### Download-src + +- `mlc` (default) + +### Download-tool + +- `r2-downloader` (default) +- `rclone` + +### Run-mode + +- `dry-run` From 997441a1e4ed4e2183c18f116ebadce489c1ae9a Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 16 Oct 2025 19:58:14 +0530 Subject: [PATCH 46/55] add tests for pointpainting (#683) --- script/get-ml-model-pointpainting/meta.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/script/get-ml-model-pointpainting/meta.yaml b/script/get-ml-model-pointpainting/meta.yaml index 8047b6188..7ed3a646f 100644 --- a/script/get-ml-model-pointpainting/meta.yaml +++ b/script/get-ml-model-pointpainting/meta.yaml @@ -86,3 +86,10 @@ variations: r2-downloader,mlc: env: MLC_DOWNLOAD_URL: https://waymo.mlcommons-storage.org/metadata/model.uri + MLC_DOWNLOAD_FILENAME: model +tests: + needs_pat: true + run_inputs: + - variations_list: + - r2-downloader,mlc,dry-run + - rclone,mlc,dry-run \ No newline at end of file From 1be4ebb22aec17d901086e403a02965029349cb3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Oct 2025 14:28:42 +0000 Subject: [PATCH 47/55] [Automated Commit] Document script/get-ml-model-pointpainting/meta.yaml [skip ci] --- script/get-ml-model-pointpainting/README.md | 81 +++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 script/get-ml-model-pointpainting/README.md diff --git a/script/get-ml-model-pointpainting/README.md b/script/get-ml-model-pointpainting/README.md new file mode 100644 index 000000000..7e59ec1e6 --- /dev/null +++ b/script/get-ml-model-pointpainting/README.md @@ -0,0 +1,81 @@ +# README for get-ml-model-pointpillars +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,ml-model,ml,model,pointpainting +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--pp_path` | | | `` | +| `--dp_resnet50_path` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Device + +- `cpu` +- `gpu` (default) + +### Download-src + +- `mlc` (default) + +### Download-tool + +- `r2-downloader` (default) +- `rclone` + +### Run-mode + +- `dry-run` From 6019630c21b6c1f85f037f815723ef3a05df6517 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 16 Oct 2025 20:48:06 +0530 Subject: [PATCH 48/55] Support r2 downloader for sdxl model download (#684) --- .../customize.py | 11 ++--- .../get-ml-model-stable-diffusion/meta.yaml | 44 ++++++++++++++++--- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/script/get-ml-model-stable-diffusion/customize.py b/script/get-ml-model-stable-diffusion/customize.py index 1eafee100..c46aab699 100644 --- a/script/get-ml-model-stable-diffusion/customize.py +++ b/script/get-ml-model-stable-diffusion/customize.py @@ -19,11 +19,12 @@ def postprocess(i): env = i['env'] - if env.get('SDXL_CHECKPOINT_PATH', '') == '': - env['SDXL_CHECKPOINT_PATH'] = env['MLC_ML_MODEL_PATH'] - elif env.get('MLC_ML_MODEL_PATH', '') == '': - env['MLC_ML_MODEL_PATH'] = env['SDXL_CHECKPOINT_PATH'] + if env.get('MLC_DOWNLOAD_MODE', '') != "dry": + if env.get('SDXL_CHECKPOINT_PATH', '') == '': + env['SDXL_CHECKPOINT_PATH'] = env['MLC_ML_MODEL_PATH'] + elif env.get('MLC_ML_MODEL_PATH', '') == '': + env['MLC_ML_MODEL_PATH'] = env['SDXL_CHECKPOINT_PATH'] - env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['SDXL_CHECKPOINT_PATH'] + env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['SDXL_CHECKPOINT_PATH'] return {'return': 0} diff --git a/script/get-ml-model-stable-diffusion/meta.yaml b/script/get-ml-model-stable-diffusion/meta.yaml index a1c96bd8b..52f8b4a7c 100644 --- a/script/get-ml-model-stable-diffusion/meta.yaml +++ b/script/get-ml-model-stable-diffusion/meta.yaml @@ -31,8 +31,8 @@ prehook_deps: force_env_keys: - MLC_OUTDIRNAME - enable_if_env: - MLC_DOWNLOAD_TOOL: - - rclone + MLC_DOWNLOAD_SOURCE: + - mlcommons MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' env: @@ -40,7 +40,7 @@ prehook_deps: extra_cache_tags: stable-diffusion,sdxl,model force_cache: true names: - - dae + - dae-sdxl-model tags: download-and-extract update_tags_from_env_with_prefix: _url.: @@ -89,7 +89,9 @@ variations: mlcommons: default: true default_variations: - download-tool: rclone + download-tool: r2-downloader + env: + MLC_DOWNLOAD_SOURCE: mlcommons group: download-source pytorch: default: true @@ -104,18 +106,33 @@ variations: required_disk_space: 13000 rclone: adr: - dae: + dae-sdxl-model: tags: _rclone env: MLC_DOWNLOAD_TOOL: rclone MLC_RCLONE_CONFIG_NAME: mlc-inference group: download-tool + r2-downloader: + adr: + dae-sdxl-model: + tags: _r2-downloader + env: + MLC_DOWNLOAD_TOOL: r2-downloader + group: download-tool rclone,fp16: env: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/stable_diffusion_fp16 rclone,fp32: env: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/stable_diffusion_fp32 + r2-downloader,fp16: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/stable-diffusion-xl-1-0-fp16-checkpoint.uri + MLC_DOWNLOAD_FILENAME: fp16-checkpoint + r2-downloader,fp32: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/stable-diffusion-xl-1-0-fp32-checkpoint.uri + MLC_DOWNLOAD_FILENAME: fp32-checkpoint uint8: env: MLC_ML_MODEL_INPUT_DATA_TYPES: uint8 @@ -124,8 +141,23 @@ variations: group: precision wget: adr: - dae: + dae-sdxl-model: tags: _wget env: MLC_DOWNLOAD_TOOL: wget group: download-tool + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x +test: + run_inputs: + - variations_list: + - fp32,r2-downloader,dry-run + - fp16,r2-downloader,dry-run From e484cd82014a27c0e6075937305ae86285386b36 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Oct 2025 15:18:36 +0000 Subject: [PATCH 49/55] [Automated Commit] Document script/get-ml-model-stable-diffusion/meta.yaml [skip ci] --- .../get-ml-model-stable-diffusion/README.md | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 script/get-ml-model-stable-diffusion/README.md diff --git a/script/get-ml-model-stable-diffusion/README.md b/script/get-ml-model-stable-diffusion/README.md new file mode 100644 index 000000000..85a3359bd --- /dev/null +++ b/script/get-ml-model-stable-diffusion/README.md @@ -0,0 +1,95 @@ +# README for get-ml-model-stable-diffusion +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,raw,ml-model,stable-diffusion,sdxl,text-to-image +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--checkpoint` | | | `` | +| `--download_path` | | | `` | +| `--to` | Alias for download_path | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Download-source + +- `huggingface` +- `mlcommons` (default) + +### Download-tool + +- `git` +- `r2-downloader` +- `rclone` +- `wget` + +### Framework + +- `pytorch` (default) + +### Precision + +- `fp16` +- `fp32` (default) +- `int8` +- `uint8` + +### Run-mode + +- `dry-run` + +### Ungrouped + +- `batch_size.#` _(# can be substituted dynamically)_ From 57e4c42a5abf132b302c2e8fbac024609073c1fc Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 16 Oct 2025 20:51:19 +0530 Subject: [PATCH 50/55] Update tags in meta.yaml for GPT-J raw model (#685) --- script/get-ml-model-gptj/meta.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/get-ml-model-gptj/meta.yaml b/script/get-ml-model-gptj/meta.yaml index f9d53576a..ea5c2dc97 100644 --- a/script/get-ml-model-gptj/meta.yaml +++ b/script/get-ml-model-gptj/meta.yaml @@ -133,7 +133,7 @@ variations: MLC_GPTJ_INTEL_MODEL: '' force_new_env_keys: - GPTJ_CHECKPOINT_PATH - tags: get,ml-model,gpt-j,_fp32,_pytorch + tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader - tags: get,conda,_name.gptj-pt - tags: get,python,_conda.gptj-pt - names: @@ -172,7 +172,7 @@ variations: - env: {} force_new_env_keys: - GPTJ_CHECKPOINT_PATH - tags: get,ml-model,gpt-j,_fp32,_pytorch + tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader - names: - nvidia-inference-common-code tags: get,nvidia,inference,common-code @@ -215,7 +215,7 @@ variations: group: framework saxml,fp32: deps: - - tags: get,ml-model,gptj,_pytorch,_fp32 + - tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader - names: - python - python3 From 5e03c9e1bde80f7c7d5a543cc64092476839ad7e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 16 Oct 2025 20:55:13 +0530 Subject: [PATCH 51/55] Changes in meta for migrating to r2 (#687) --- .../meta.yaml | 19 ++++++------------ script/app-mlperf-inference-nvidia/meta.yaml | 4 ++-- script/app-mlperf-inference-redhat/meta.yaml | 2 +- script/app-mlperf-inference/meta.yaml | 20 +++++++++---------- .../meta.yaml | 2 +- script/get-ml-model-dlrm-terabyte/meta.yaml | 2 +- .../get-preprocessed-dataset-criteo/meta.yaml | 4 ++-- script/process-mlperf-accuracy/meta.yaml | 10 +++++----- 8 files changed, 28 insertions(+), 35 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index 14b29fc79..85ab46001 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -372,7 +372,7 @@ deps: - retinanet ## GPT-J - - tags: get,ml-model,large-language-model,gptj + - tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader names: - ml-model - gptj-model @@ -450,7 +450,7 @@ deps: - "yes" ## mixtral-8x7b - - tags: get,ml-model,mixtral + - tags: get,ml-model,mixtral,_r2-downloader,_mlc names: - ml-model - mixtral-model @@ -542,7 +542,7 @@ deps: - "yes" ## pointpainting - - tags: get,ml-model,pointpainting + - tags: get,ml-model,pointpainting,_r2-downloader,_mlc names: - pointpillars-model enable_if_env: @@ -692,7 +692,7 @@ deps: - "yes" ## waymo for pointpillats - - tags: get,dataset,waymo + - tags: get,dataset,waymo,_r2-downloader,_mlc enable_if_env: MLC_MODEL: - pointpainting @@ -1464,13 +1464,6 @@ variations: - tags: get,generic-python-lib,_package.evaluate - tags: get,generic-python-lib,_package.absl-py - tags: get,generic-python-lib,_package.rouge-score - - tags: get,ml-model,llama3,_hf,_meta-llama/Llama-3.1-8B-Instruct - skip_if_env: - MLC_RUN_STATE_DOCKER: - - "yes" - names: - - llama3_1-8b - - llama3-8b llama3_1-8b: base: @@ -1479,7 +1472,7 @@ variations: MLC_MODEL: llama3_1-8b deps: ## CNNDM for Llama3 8B model - datacenter - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3 + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-datacenter skip_if_env: @@ -1493,7 +1486,7 @@ variations: MLC_MODEL: llama3_1-8b-edge deps: ## CNNDM for Llama3 8B model - edge - - tags: get,dataset,cnndm,_validation,_edge,_llama3 + - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-edge skip_if_env: diff --git a/script/app-mlperf-inference-nvidia/meta.yaml b/script/app-mlperf-inference-nvidia/meta.yaml index c2f9b3bdc..09af92c8b 100644 --- a/script/app-mlperf-inference-nvidia/meta.yaml +++ b/script/app-mlperf-inference-nvidia/meta.yaml @@ -306,7 +306,7 @@ prehook_deps: - preprocess_data names: - gptj-model - tags: get,ml-model,gptj,_pytorch,_rclone + tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader # Download model for sdxl - enable_if_env: @@ -318,7 +318,7 @@ prehook_deps: - stable-diffusion-xl - sdxl-model - ml-model - tags: get,ml-model,sdxl,_fp16,_rclone + tags: get,ml-model,sdxl,_fp16,_r2-downloader skip_if_env: MLC_RUN_STATE_DOCKER: - 'yes' diff --git a/script/app-mlperf-inference-redhat/meta.yaml b/script/app-mlperf-inference-redhat/meta.yaml index 452e8094e..d02b964bd 100644 --- a/script/app-mlperf-inference-redhat/meta.yaml +++ b/script/app-mlperf-inference-redhat/meta.yaml @@ -221,7 +221,7 @@ variations: gptj_: deps: - - tags: get,ml-model,gptj + - tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader names: - gptj-model - tags: get,dataset,cnndm,_validation diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index 56fc83128..feb8b7452 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -317,7 +317,7 @@ variations: all-models,nvidia-original: docker: deps: - - tags: get,ml-model,gptj,raw + - tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader skip_if_env: MLC_MLPERF_NVIDIA_SKIP_GPTJ: - "yes" @@ -1036,13 +1036,13 @@ variations: pointpainting docker: deps: - - tags: get,dataset,waymo + - tags: get,dataset,waymo,_r2-downloader,_mlc enable_if_env: MLC_USE_DATASET_FROM_HOST: - 'yes' names: - waymo-dataset - - tags: get,ml-model,pointpainting + - tags: get,ml-model,pointpainting,_r2-downloader,_mlc enable_if_env: MLC_USE_ML_MODEL_FROM_HOST: - 'yes' @@ -1231,7 +1231,7 @@ variations: names: - llama3_1-405b - llama3-405b - - tags: mlcr get,dataset,mlperf,inference,llama3,_validation,_r2-downloader + - tags: get,dataset,mlperf,inference,llama3,_validation,_r2-downloader enable_if_env: MLC_USE_DATASET_FROM_HOST: - 'yes' @@ -1316,7 +1316,7 @@ variations: - 'yes' MLC_USE_MODEL_FROM_HOST: - 'yes' - tags: get,ml-model,sdxl,_fp16,_rclone + tags: get,ml-model,sdxl,_fp16,_r2-downloader sdxl,reference,float16: docker: @@ -1327,7 +1327,7 @@ variations: - 'yes' MLC_USE_MODEL_FROM_HOST: - 'yes' - tags: get,ml-model,sdxl,_fp16,_rclone + tags: get,ml-model,sdxl,_fp16,_r2-downloader sdxl,reference,bfloat16: docker: @@ -1338,7 +1338,7 @@ variations: - 'yes' MLC_USE_MODEL_FROM_HOST: - 'yes' - tags: get,ml-model,sdxl,_fp16,_rclone + tags: get,ml-model,sdxl,_fp16,__r2-downloader sdxl,reference,float32: docker: @@ -1349,7 +1349,7 @@ variations: - 'yes' MLC_USE_MODEL_FROM_HOST: - 'yes' - tags: get,ml-model,sdxl,_fp32,_rclone + tags: get,ml-model,sdxl,_fp32,__r2-downloader llama2-70b_: default_env: @@ -1463,7 +1463,7 @@ variations: docker: image_name: mlperf-inference-mlcommons-python-implementation-mixtral-8x7b deps: - - tags: get,ml-model,mixtral + - tags: get,ml-model,mixtral,_r2-downloader,_mlc names: - ml-model - mixtral-model @@ -1540,7 +1540,7 @@ variations: gptj_: docker: deps: - - tags: get,ml-model,gptj,raw + - tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader env: MLC_MLPERF_MODEL_EQUAL_ISSUE_MODE: 'yes' posthook_deps: diff --git a/script/get-dataset-mlperf-inference-mixtral/meta.yaml b/script/get-dataset-mlperf-inference-mixtral/meta.yaml index 442510ebd..a79b70dac 100644 --- a/script/get-dataset-mlperf-inference-mixtral/meta.yaml +++ b/script/get-dataset-mlperf-inference-mixtral/meta.yaml @@ -26,7 +26,6 @@ uid: 89e7c91444804775 variations: rclone: group: download-tool - default: true adr: download-file-mixtral-dataset: tags: _rclone @@ -35,6 +34,7 @@ variations: MLC_RCLONE_COPY_USING: copyurl r2-downloader: group: download-tool + default: true adr: download-file-mixtral-dataset: tags: _r2-downloader diff --git a/script/get-ml-model-dlrm-terabyte/meta.yaml b/script/get-ml-model-dlrm-terabyte/meta.yaml index 372960045..86b6bfa90 100644 --- a/script/get-ml-model-dlrm-terabyte/meta.yaml +++ b/script/get-ml-model-dlrm-terabyte/meta.yaml @@ -93,7 +93,7 @@ variations: MLC_PACKAGE_URL: https://dlrm.s3-us-west-1.amazonaws.com/models/tb0875_10M.pt pytorch,fp32,weight_sharded: default_variations: - download-tool: rclone + download-tool: r2-downloader env: MLC_DOWNLOAD_CHECKSUM: '' MLC_ML_MODEL_ACCURACY: '0.8025' diff --git a/script/get-preprocessed-dataset-criteo/meta.yaml b/script/get-preprocessed-dataset-criteo/meta.yaml index 56e1d98bd..0fa577d02 100644 --- a/script/get-preprocessed-dataset-criteo/meta.yaml +++ b/script/get-preprocessed-dataset-criteo/meta.yaml @@ -137,7 +137,6 @@ variations: multihot,mlc: deps: - env: - MLC_DOWNLOAD_CHECKSUM_FILE: <<>>/checksums.txt MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_PREPROCESSED_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_PREPROCESSED_PATH MLC_EXTRACT_TO_FOLDER: criteo-preprocessed @@ -153,6 +152,7 @@ variations: - MLC_DOWNLOAD_URL multihot,mlc,rclone: env: + MLC_DOWNLOAD_CHECKSUM_FILE: <<>>/checksums.txt MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/dlrm_preprocessed MLC_RCLONE_CONFIG_NAME: mlc-inference multihot,mlc,r2-downloader: @@ -163,12 +163,12 @@ variations: add_deps_recursive: dae-criteo-preprocessed: tags: _rclone - default: true r2-downloader: group: download-tool add_deps_recursive: dae-criteo-preprocessed: tags: _r2-downloader + default: true dry-run: group: run-mode env: diff --git a/script/process-mlperf-accuracy/meta.yaml b/script/process-mlperf-accuracy/meta.yaml index 24ff6fdd0..56fe7d5a0 100644 --- a/script/process-mlperf-accuracy/meta.yaml +++ b/script/process-mlperf-accuracy/meta.yaml @@ -94,7 +94,7 @@ variations: - enable_if_env: MLC_MLPERF_IMPLEMENTATION: - intel - tags: get,ml-model,gptj,_fp32,_pytorch + tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader env: MLC_DATASET: cnndm group: dataset @@ -235,7 +235,7 @@ variations: skip_if_env: MLC_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST: - 'yes' - tags: get,ml-model,mixtral + tags: get,ml-model,mixtral,_r2-downloader,_mlc env: MLC_DATASET: openorca-gsm8k-mbxp-combined group: dataset @@ -340,7 +340,7 @@ variations: group: dataset cnndm_llama_3: deps: - - tags: get,ml-model,llama3,_hf,_meta-llama/Llama-3.1-8B-Instruct + - tags: get,ml-model,llama3,_8b names: - llama3_1-8b - llama3-8b @@ -353,7 +353,7 @@ variations: group: dataset cnndm_llama_3,edge: deps: - - tags: get,dataset,cnndm,_validation,_edge,_llama3 + - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-edge skip_if_env: @@ -361,7 +361,7 @@ variations: - "yes" cnndm_llama_3,datacenter: deps: - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3 + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-datacenter skip_if_env: From b0af18e90e6b84dfbf150809b6feb47139eec6ba Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Oct 2025 15:25:44 +0000 Subject: [PATCH 52/55] [Automated Commit] Document script/process-mlperf-accuracy/meta.yaml [skip ci] --- script/process-mlperf-accuracy/README.md | 104 +++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 script/process-mlperf-accuracy/README.md diff --git a/script/process-mlperf-accuracy/README.md b/script/process-mlperf-accuracy/README.md new file mode 100644 index 000000000..823fd4a9f --- /dev/null +++ b/script/process-mlperf-accuracy/README.md @@ -0,0 +1,104 @@ +# README for process-mlperf-accuracy +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--rerun` | | | `` | +| `--result_dir` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Category + +- `datacenter` +- `edge` + +### Coco-evaluation-tool + +- `default-pycocotools` (default) +- `nvidia-pycocotools` + +### Dataset + +- `cnndm` +- `cnndm_llama_3` +- `coco2014` +- `cognata_deeplab` +- `cognata_ssd` +- `dataset_llama3` +- `igbh` +- `imagenet` (default) +- `kits19` +- `librispeech` +- `librispeech_whisper` +- `nuscenes` +- `open-orca` +- `openimages` +- `openorca-gsm8k-mbxp` +- `squad` +- `terabyte` +- `waymo` + +### Precision + +- `float16` +- `float32` (default) +- `float64` +- `int16` +- `int32` +- `int64` +- `int8` From 8fa20590bc8be1338bde852ef17418151375427a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 20 Oct 2025 01:44:39 +0100 Subject: [PATCH 53/55] Fixes for llvm installation from src (#690) --- script/download-and-extract/customize.py | 3 +++ script/download-and-extract/meta.yaml | 3 +++ script/install-llvm-src/customize.py | 8 +++++++- script/install-llvm-src/meta.yaml | 7 +++++++ script/install-llvm-src/run.sh | 4 +++- 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/script/download-and-extract/customize.py b/script/download-and-extract/customize.py index 4a11b4fd5..bc3d2a6ad 100644 --- a/script/download-and-extract/customize.py +++ b/script/download-and-extract/customize.py @@ -48,6 +48,9 @@ def preprocess(i): env.get('MLC_DAE_FINAL_ENV_NAME', '') != ''): env['MLC_EXTRACT_FINAL_ENV_NAME'] = env['MLC_DAE_FINAL_ENV_NAME'] + if is_true(env.get('MLC_DAE_EXTRACT_TO_DOWNLOADED')): + env['MLC_OUTDIRNAME'] = os.getcwd() + return {'return': 0} diff --git a/script/download-and-extract/meta.yaml b/script/download-and-extract/meta.yaml index 85fcd25ac..e031655d5 100644 --- a/script/download-and-extract/meta.yaml +++ b/script/download-and-extract/meta.yaml @@ -78,6 +78,9 @@ variations: extract: env: MLC_DAE_EXTRACT_DOWNLOADED: 'yes' + extract-to-download-dir: + env: + MLC_DAE_EXTRACT_TO_DOWNLOADED: 'yes' gdown: add_deps: download-script: diff --git a/script/install-llvm-src/customize.py b/script/install-llvm-src/customize.py index 12680c47e..9a82d0c37 100644 --- a/script/install-llvm-src/customize.py +++ b/script/install-llvm-src/customize.py @@ -22,6 +22,9 @@ def preprocess(i): else: cmake_options = '' + if not is_true(env.get('MLC_LLVM_PER_TARGET_RUNTIME_DIR')): + cmake_options += " -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF " + extra_cmake_options = cmake_options + \ env.get('MLC_LLVM_EXTRA_CMAKE_OPTIONS', '') @@ -31,6 +34,9 @@ def preprocess(i): else: install_prefix = os.path.join(os.getcwd(), "install") + if is_true(env.get('MLC_CLEAN')): + env['MLC_CLEAN_BUILD'] = 'yes' + if os.path.exists(os.path.join(install_prefix, "bin", "clang")) and is_true( env.get('MLC_LLVM_USE_INSTALLED_DIR')): i['run_script_input']['script_name'] = "no-install" # skip install @@ -61,7 +67,7 @@ def preprocess(i): targets_to_build = env.get('MLC_LLVM_TARGETS_TO_BUILD') host_platform = env.get('MLC_HOST_PLATFORM_FLAVOR') if not targets_to_build: - if 'arm64' in host_platform: + if 'arm64' in host_platform or 'aarch64' in host_platform: targets_to_build = 'AArch64' else: targets_to_build = 'X86' diff --git a/script/install-llvm-src/meta.yaml b/script/install-llvm-src/meta.yaml index 42189a7d4..1e48dae73 100644 --- a/script/install-llvm-src/meta.yaml +++ b/script/install-llvm-src/meta.yaml @@ -40,6 +40,8 @@ input_mapping: targets: MLC_LLVM_TARGETS_TO_BUILD target_triple: MLC_LLVM_TARGET_TRIPLE extra_options: MLC_LLVM_EXTRA_CMAKE_OPTIONS + per_target_runtime_dir: MLC_LLVM_PER_TARGET_RUNTIME_DIR + clean: MLC_CLEAN env: MLC_GIT_URL: https://github.com/llvm/llvm-project name: Build LLVM compiler from sources (can take >30 min) @@ -145,8 +147,13 @@ variations: - tags: get,generic-sys-util,_crossbuild-essential-arm64 - tags: get,generic-sys-util,_gfortran-12-aarch64-linux-gnu + default_env: + MLC_LLVM_TARGETS_TO_BUILD: AArch64 env: MLC_LLVM_CROSS_COMPILE_FLAGS: "-DCMAKE_SYSTEM_NAME=Linux -DCMAKE_SYSTEM_PROCESSOR=aarch64 -DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-linux-gnu -DCMAKE_Fortran_COMPILER=aarch64-linux-gnu-gfortran-12 -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++" + cross-compile-x86-aarch64,flang: + deps: + - tags: get,generic-sys-util,_gfortran-12-aarch64-linux-gnu for-intel-mlperf-inference-v3.1-bert: adr: conda-package: diff --git a/script/install-llvm-src/run.sh b/script/install-llvm-src/run.sh index 52178a27a..f23892265 100644 --- a/script/install-llvm-src/run.sh +++ b/script/install-llvm-src/run.sh @@ -14,7 +14,9 @@ else fi if [[ ${MLC_CLEAN_BUILD} == "yes" ]]; then - rm -rf build + cmd="rm -rf build" + echo "$cmd" + eval "$cmd" fi mkdir -p build From 14d8452d7d7c6c57a40406a52e325ec9dd2ff5d0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 20 Oct 2025 00:45:29 +0000 Subject: [PATCH 54/55] [Automated Commit] Document script/download-and-extract/meta.yaml [skip ci] --- script/download-and-extract/README.md | 90 +++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 script/download-and-extract/README.md diff --git a/script/download-and-extract/README.md b/script/download-and-extract/README.md new file mode 100644 index 000000000..09a080c89 --- /dev/null +++ b/script/download-and-extract/README.md @@ -0,0 +1,90 @@ +# README for download-and-extract +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr download-and-extract file +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--download_path` | | | `` | +| `--extra_folder` | | | `` | +| `--extract_path` | | | `` | +| `--from` | | | `` | +| `--local_path` | Alias for from | | `` | +| `--store` | Alias for download_path | | `` | +| `--to` | Alias for extract_path | | `` | +| `--url` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Download-tool + +- `curl` +- `gdown` +- `mlcutil` (alias: cmutil) (default) +- `r2-downloader` +- `rclone` +- `torrent` +- `wget` + +### Keep + +- `keep` (default) +- `remove-extracted` + +### Ungrouped + +- `extract` +- `extract-to-download-dir` +- `url.#` _(# can be substituted dynamically)_ From cc62f3b69946c4a1ae4d80fdeee70a801aecd14e Mon Sep 17 00:00:00 2001 From: amd-arsuresh Date: Mon, 20 Oct 2025 19:27:58 +0100 Subject: [PATCH 55/55] Refactor update_state_from_variations, support dynamic variation in combination of variations (#691) * Refactor update_state_from_variations * Support dynamic variation inside combination of variations --- .github/workflows/build_wheel_off.yml | 1 + automation/script/module.py | 446 +++++++++++++++----------- 2 files changed, 267 insertions(+), 180 deletions(-) diff --git a/.github/workflows/build_wheel_off.yml b/.github/workflows/build_wheel_off.yml index 4fbc24dba..199c11d54 100644 --- a/.github/workflows/build_wheel_off.yml +++ b/.github/workflows/build_wheel_off.yml @@ -7,6 +7,7 @@ on: paths: - VERSION + jobs: build_wheels: if: github.repository_owner == 'mlcommons' diff --git a/automation/script/module.py b/automation/script/module.py index a815bdd38..bf37ab0eb 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -7,6 +7,7 @@ # Developed by Grigori Fursin and Arjun Suresh for CM and modified for MLCFlow by Arjun Suresh and Anandhu Sooraj # +import re import os import logging @@ -872,13 +873,6 @@ def _run(self, i): prehook_deps = meta.get('prehook_deps', []) posthook_deps = meta.get('posthook_deps', []) - # Store the default_version in run_state -> may be overridden by - # variations - default_version = meta.get( - 'default_version', - '') # not used if version is given - run_state['default_version'] = default_version - # STEP 700: Overwrite env with keys from the script input (to allow user friendly CLI) # IT HAS THE PRIORITY OVER meta['default_env'] and meta['env'] but not over the meta from versions/variations # (env OVERWRITE - user enforces it from CLI) @@ -1471,7 +1465,7 @@ def _run(self, i): # Update default version meta if version is not set if version == '': - default_version = run_state.get('default_version', '') + default_version = meta.get('default_version', '') if default_version != '': version = default_version @@ -2345,244 +2339,310 @@ def _dump_version_info_for_script( return {'return': 0} - ########################################################################## - def _update_state_from_variations(self, i, meta, variation_tags, variations, env, state, const, const_state, deps, post_deps, prehook_deps, - posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, add_deps_recursive, run_state, recursion_spaces): - - logger = self.action_object.logger - # Save current explicit variations + def _update_state_from_variations( + self, i, meta, variation_tags, variations, env, state, + const, const_state, deps, post_deps, prehook_deps, + posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, + add_deps_recursive, run_state, recursion_spaces + ): import copy - explicit_variation_tags = copy.deepcopy(variation_tags) + logger = self.action_object.logger + explicit_variation_tags = copy.deepcopy(variation_tags) if add_deps_recursive is None: add_deps_recursive = {} - - # Calculate space required_disk_space = {} - - # Check if warning warnings = [] - # variation_tags get appended by any aliases + # 1️⃣ Expand and validate variation tags r = self._get_variations_with_aliases(variation_tags, variations) if r['return'] > 0: return r variation_tags = r['variation_tags'] excluded_variation_tags = r['excluded_variation_tags'] - # Get a dictionary of variation groups r = self._get_variation_groups(variations) if r['return'] > 0: return r - variation_groups = r['variation_groups'] - run_state['variation_groups'] = variation_groups - # Add variation(s) if specified in the "tags" input prefixed by _ - # If there is only 1 default variation, then just use it or - # substitute from CMD - default_variation = meta.get('default_variation', '') - if default_variation and default_variation not in variations: - return {'return': 1, 'error': 'Default variation "{}" is not in the list of variations: "{}" '.format( - default_variation, variations.keys())} + return { + 'return': 1, 'error': f'Default variation "{default_variation}" not in {list(variations.keys())}'} - if len(variation_tags) == 0: - if default_variation != '' and default_variation not in excluded_variation_tags: - variation_tags = [default_variation] + if not variation_tags and default_variation and default_variation not in excluded_variation_tags: + variation_tags = [default_variation] r = self._update_variation_tags_from_variations( - variation_tags, variations, variation_groups, excluded_variation_tags) + variation_tags, variation_groups, excluded_variation_tags, variations) if r['return'] > 0: return r - # variation_tags get appended by any default on variation in groups + # process group defaults r = self._process_variation_tags_in_groups( variation_tags, variation_groups, excluded_variation_tags, variations) if r['return'] > 0: return r if variation_tags != r['variation_tags']: variation_tags = r['variation_tags'] - # we need to again process variation tags if any new default # variation is added r = self._update_variation_tags_from_variations( - variation_tags, variations, variation_groups, excluded_variation_tags) + variation_tags, variation_groups, excluded_variation_tags, variations) if r['return'] > 0: return r - valid_variation_combinations = meta.get( - 'valid_variation_combinations', []) - if valid_variation_combinations: - if not any(all(t in variation_tags for t in s) - for s in valid_variation_combinations): - return {'return': 1, 'error': 'Invalid variation combination "{}" prepared. Valid combinations: "{}" '.format( - variation_tags, valid_variation_combinations)} - - invalid_variation_combinations = meta.get( - 'invalid_variation_combinations', []) - if invalid_variation_combinations: - if any(all(t in variation_tags for t in s) - for s in invalid_variation_combinations): - return {'return': 1, 'error': 'Invalid variation combination "{}" prepared. Invalid combinations: "{}" '.format( - variation_tags, invalid_variation_combinations)} + # Validate combinations + r = self._validate_variations(meta, variation_tags) + if r['return'] > 0: + return r variation_tags_string = '' - if len(variation_tags) > 0: - for t in variation_tags: - if variation_tags_string != '': - variation_tags_string += ',' - - x = '_' + t - variation_tags_string += x + if variation_tags: + variation_tags_string = ','.join(['_' + t for t in variation_tags]) logger.debug( - recursion_spaces + - ' Prepared variations: {}'.format(variation_tags_string)) + f"{recursion_spaces}Prepared variations: {variation_tags_string}") - # Update env and other keys if variations - if len(variation_tags) > 0: + # 2️⃣ Apply individual variations for variation_tag in variation_tags: - if variation_tag.startswith('~'): - # ignore such tag (needed for caching only to differentiate - # variations) - continue - - if variation_tag.startswith('-'): - # ignore such tag (needed for caching only to eliminate - # variations) - continue - - variation_tag_dynamic_suffix = None - if variation_tag not in variations: - if '.' in variation_tag and variation_tag[-1] != '.': - variation_tag_dynamic_suffix = variation_tag[variation_tag.index( - ".") + 1:] - if not variation_tag_dynamic_suffix: - return {'return': 1, 'error': 'tag {} is not in variations {}'.format( - variation_tag, variations.keys())} - variation_tag = self._get_name_for_dynamic_variation_tag( - variation_tag) - if variation_tag not in variations: - return {'return': 1, 'error': 'tag {} is not in variations {}'.format( - variation_tag, variations.keys())} - - variation_meta = variations[variation_tag] - if variation_tag_dynamic_suffix: - variation_meta = copy.deepcopy(variation_meta) - self._update_variation_meta_with_dynamic_suffix( - variation_meta, variation_tag_dynamic_suffix) - - r = update_state_from_meta( - variation_meta, - env, - state, - const, - const_state, - deps, - post_deps, - prehook_deps, - posthook_deps, - new_env_keys_from_meta, - new_state_keys_from_meta, - run_state, - i) + r = self._apply_single_variation( + variation_tag, variations, env, state, const, const_state, + deps, post_deps, prehook_deps, posthook_deps, + new_env_keys_from_meta, new_state_keys_from_meta, + run_state, i, meta, required_disk_space, warnings, add_deps_recursive + ) if r['return'] > 0: return r - if variation_meta.get('script_name', '') != '': - meta['script_name'] = variation_meta['script_name'] + # 3️⃣ Apply combined variations + r = self._apply_combined_variations( + variations, variation_tags, env, state, const, const_state, + deps, post_deps, prehook_deps, posthook_deps, + new_env_keys_from_meta, new_state_keys_from_meta, + run_state, i, meta, required_disk_space, warnings, add_deps_recursive + ) + if r['return'] > 0: + return r - if variation_meta.get('default_version', '') != '': - run_state['default_version'] = variation_meta['default_version'] + # Processing them again using updated deps for add_deps_recursive + r = update_adr_from_meta( + deps, + post_deps, + prehook_deps, + posthook_deps, + add_deps_recursive, + env) + if r['return'] > 0: + return r - if variation_meta.get( - 'required_disk_space', 0) > 0 and variation_tag not in required_disk_space: - required_disk_space[variation_tag] = variation_meta['required_disk_space'] + # Done + return { + 'return': 0, + 'variation_tags_string': variation_tags_string, + 'explicit_variation_tags': explicit_variation_tags, + 'warnings': warnings, + 'required_disk_space': required_disk_space, + 'variation_tags': variation_tags + } - if variation_meta.get('warning', '') != '': - x = variation_meta['warning'] - if x not in warnings: - warnings.append() + def _apply_variation_meta(self, variation_key, variation_meta, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, + new_env_keys_from_meta, new_state_keys_from_meta, run_state, i, meta, required_disk_space, warnings, add_deps_recursive): + r = update_state_from_meta( + variation_meta, + env, + state, + const, + const_state, + deps, + post_deps, + prehook_deps, + posthook_deps, + new_env_keys_from_meta, + new_state_keys_from_meta, + run_state, + i) + if r['return'] > 0: + return r - adr = get_adr(variation_meta) - if adr: - self._merge_dicts_with_tags(add_deps_recursive, adr) + adr = get_adr(variation_meta) + if adr: + self._merge_dicts_with_tags(add_deps_recursive, adr) - combined_variations = [t for t in variations if ',' in t] + # Merge selected meta keys + for k in ('script_name', 'default_version'): + if k in variation_meta: + meta[k] = variation_meta[k] - combined_variations.sort(key=lambda x: x.count(',')) - ''' By sorting based on the number of variations users can safely override - env and state in a larger combined variation - ''' + # Track required disk space per variation key + rds = variation_meta.get('required_disk_space', 0) + if rds > 0: + required_disk_space[variation_key] = rds - for combined_variation in combined_variations: - v = combined_variation.split(",") - all_present = set(v).issubset(set(variation_tags)) - if all_present: + # Collect unique warnings + x = variation_meta.get('warning') + if x and x not in warnings: + warnings.append(x) - combined_variation_meta = variations[combined_variation] + return {'return': 0} - r = update_state_from_meta( - combined_variation_meta, - env, - state, - const, - const_state, - deps, - post_deps, - prehook_deps, - posthook_deps, - new_env_keys_from_meta, - new_state_keys_from_meta, - run_state, - i) - if r['return'] > 0: - return r + def _resolve_dynamic_tag(self, tag, variations): + # Returns (base_tag, suffix) for a possibly dynamic tag. # If not + # dynamic or cannot be resolved, returns (tag, None). + + if '.' in tag and tag[-1] != '.': + suffix = tag.split('.', 1)[1] + base = self._get_name_for_dynamic_variation_tag(tag) + + # Only accept if base is a known variation key; otherwise treat as + # non-dynamic + if base in variations: + return base, suffix + return tag, None + + def _apply_single_variation( + self, variation_tag, variations, env, state, const, const_state, + deps, post_deps, prehook_deps, posthook_deps, + new_env_keys_from_meta, new_state_keys_from_meta, + run_state, i, meta, required_disk_space, warnings, add_deps_recursive + ): + import copy + if variation_tag.startswith(('~', '-')): + return {'return': 0} - adr = get_adr(combined_variation_meta) - if adr: - self._merge_dicts_with_tags( - add_deps_recursive, adr) + variation_tag_dynamic_suffix = None + if variation_tag not in variations: + if '.' in variation_tag and variation_tag[-1] != '.': + variation_tag_dynamic_suffix = variation_tag.split('.', 1)[1] + variation_tag = self._get_name_for_dynamic_variation_tag( + variation_tag) + if variation_tag not in variations: + return { + 'return': 1, + 'error': f'Tag {variation_tag} not in variations {list(variations.keys())}' + } - if combined_variation_meta.get( - 'script_name', '') != '': - meta['script_name'] = combined_variation_meta['script_name'] + variation_meta = variations[variation_tag] + if variation_tag_dynamic_suffix: + variation_meta = copy.deepcopy(variation_meta) + self._update_variation_meta_with_dynamic_suffix( + variation_meta, variation_tag_dynamic_suffix + ) + + return self._apply_variation_meta( + variation_key=variation_tag, + variation_meta=variation_meta, + env=env, state=state, const=const, const_state=const_state, + deps=deps, post_deps=post_deps, prehook_deps=prehook_deps, posthook_deps=posthook_deps, + new_env_keys_from_meta=new_env_keys_from_meta, new_state_keys_from_meta=new_state_keys_from_meta, + run_state=run_state, i=i, + meta=meta, + required_disk_space=required_disk_space, + warnings=warnings, + add_deps_recursive=add_deps_recursive + ) + + def _is_dynamic_placeholder(self, comp): + return isinstance(comp, str) and comp.endswith('.#') + + def _dynamic_base(self, comp): # Assumes comp endswith ".#" + return comp[:-2] + + def _apply_combined_variations( + self, variations, variation_tags, env, state, const, const_state, + deps, post_deps, prehook_deps, posthook_deps, + new_env_keys_from_meta, new_state_keys_from_meta, + run_state, i, meta, required_disk_space, warnings, add_deps_recursive + ): - if variation_meta.get('default_version', '') != '': - run_state['default_version'] = variation_meta['default_version'] + # Only consider string keys that are combined (contain a comma) + combined_variations = [ + t for t in variations if isinstance( + t, str) and ',' in t] + # Ensure we apply more specific (more components) first + combined_variations.sort(key=lambda x: x.count(','), reverse=False) + + # Pre-resolve selected tags into base->(original_tag, suffix) so we can match combined keys by base + # If multiple selected tags map to the same base, keep the first + # occurrence + selected_by_base = {} + for sel in variation_tags: + base, suffix = self._resolve_dynamic_tag(sel, variations) + if base not in selected_by_base: + selected_by_base[base] = (sel, suffix) + + for combined_key in combined_variations: + components = combined_key.split(',') + + # Split into dynamic placeholders and static components + dyn_components = [ + c for c in components if self._is_dynamic_placeholder(c)] + static_components = [ + c for c in components if not self._is_dynamic_placeholder(c)] + + # For static components, require presence via relaxed_subset + # (so a selected "blas.3" can satisfy static "blas" if relaxed_subset allows it) + static_ok = set(static_components).issubset(set(variation_tags)) + if not static_ok: + continue - if combined_variation_meta.get( - 'required_disk_space', 0) > 0 and combined_variation not in required_disk_space: - required_disk_space[combined_variation] = combined_variation_meta['required_disk_space'] + # For dynamic placeholders, require the base to be present in + # selected tags + dyn_bases = [self._dynamic_base(c) + '.#' for c in dyn_components] + if not all(b in selected_by_base for b in dyn_bases): + continue - if combined_variation_meta.get('warning', '') != '': - x = combined_variation_meta['warning'] - if x not in warnings: - warnings.append(x) + # Enforce at most one dynamic component among these bases + dynamic_infos = [(b, selected_by_base[b][1]) for b in dyn_bases] + dynamic_infos = [(b, sfx) for b, sfx in dynamic_infos if sfx] + if len(dynamic_infos) > 1: + # Skip if more than one dynamic suffix is present + continue - # Processing them again using updated deps for add_deps_recursive - r = update_adr_from_meta( - deps, - post_deps, - prehook_deps, - posthook_deps, - add_deps_recursive, - env) + combined_meta = variations[combined_key] + + # If exactly one dynamic suffix among components, apply it to a + # copy of the combined meta + if len(dynamic_infos) == 1: + _, dyn_suffix = dynamic_infos[0] + import copy + combined_meta = copy.deepcopy(combined_meta) + self._update_variation_meta_with_dynamic_suffix( + combined_meta, dyn_suffix) + + r = self._apply_variation_meta( + variation_key=combined_key, + variation_meta=combined_meta, + env=env, state=state, const=const, const_state=const_state, + deps=deps, post_deps=post_deps, prehook_deps=prehook_deps, posthook_deps=posthook_deps, + new_env_keys_from_meta=new_env_keys_from_meta, new_state_keys_from_meta=new_state_keys_from_meta, + run_state=run_state, i=i, + meta=meta, + required_disk_space=required_disk_space, + warnings=warnings, + add_deps_recursive=add_deps_recursive + ) if r['return'] > 0: return r - if len(required_disk_space) > 0: - required_disk_space_sum_mb = sum( - list(required_disk_space.values())) + return {'return': 0} + + def _validate_variations(self, meta, variation_tags): + valid = meta.get('valid_variation_combinations', []) + if valid: + if not any(all(t in variation_tags for t in s) for s in valid): + return { + 'return': 1, 'error': f'Invalid combination {variation_tags}, valid: {valid}'} - warnings.append( - 'Required disk space: {} MB'.format(required_disk_space_sum_mb)) + invalid = meta.get('invalid_variation_combinations', []) + if invalid: + if any(all(t in variation_tags for t in s) for s in invalid): + return { + 'return': 1, 'error': f'Combination {variation_tags} is explicitly invalid: {invalid}'} - return {'return': 0, 'variation_tags_string': variation_tags_string, - 'explicit_variation_tags': explicit_variation_tags, 'warnings': warnings} + return {'return': 0} def _add_base_variations( self, @@ -2642,7 +2702,7 @@ def _add_base_variations( ########################################################################## def _update_variation_tags_from_variations( - self, variation_tags, variations, variation_groups, excluded_variation_tags): + self, variation_tags, variation_groups, excluded_variation_tags, variations): import copy tmp_variation_tags_static = copy.deepcopy(variation_tags) @@ -2713,7 +2773,7 @@ def _update_variation_tags_from_variations( if tmp_combined_variations[combined_variation]: continue v = combined_variation.split(",") - all_present = set(v).issubset(set(variation_tags)) + all_present = relaxed_subset(v, variation_tags) if all_present: combined_variation_meta = variations[combined_variation] tmp_combined_variations[combined_variation] = True @@ -2757,6 +2817,7 @@ def _update_variation_tags_from_variations( return {'return': 0} ########################################################################## + def _get_variation_tags_from_default_variations( self, variation_meta, variations, variation_groups, tmp_variation_tags_static, excluded_variation_tags): # default_variations dictionary specifies the default_variation for @@ -4696,6 +4757,31 @@ def _select_script(self, i): 'script': selected_artifact } +########################################################################## + + +def relaxed_subset(v, variation_tags): + """ + Returns True if every element in v matches at least one element + in variation_tags, allowing patterns like 'build.#' to match + anything that starts with 'build.'. + """ + for item in v: + matched = False + for tag in variation_tags: + if item == tag: + matched = True + break + # Relaxed match: 'build.#' -> matches 'build.' + anything + if item.endswith(".#"): + prefix = item[:-2] + if tag.startswith(prefix + "."): + matched = True + break + if not matched: + return False + return True + def get_version_tag_from_version(version, cached_tags): tags_to_add = []