diff --git a/.github/workflows/build_wheel_off.yml b/.github/workflows/build_wheel_off.yml index 4fbc24dba..199c11d54 100644 --- a/.github/workflows/build_wheel_off.yml +++ b/.github/workflows/build_wheel_off.yml @@ -7,6 +7,7 @@ on: paths: - VERSION + jobs: build_wheels: if: github.repository_owner == 'mlcommons' diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 55e79c0ae..86b93ac06 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -5,7 +5,6 @@ on: types: [published] workflow_dispatch: {} - jobs: build_wheels: if: github.repository_owner == 'mlcommons' diff --git a/.github/workflows/test-mlperf-inference-submission-generation.yml b/.github/workflows/test-mlperf-inference-submission-generation.yml index a0cb0daec..c66beaafd 100644 --- a/.github/workflows/test-mlperf-inference-submission-generation.yml +++ b/.github/workflows/test-mlperf-inference-submission-generation.yml @@ -29,42 +29,13 @@ jobs: python-version: [ "3.12" ] division: ["closed", "open", "closed-open"] category: ["datacenter", "edge"] - round: ["5.0", "5.1"] + round: ["5.1"] folder_name: ["closed", "open"] action: ["run", "docker"] exclude: - os: macos-latest - os: windows-latest - category: "edge" - include: - - os: ubuntu-latest - python-version: "3.12" - division: "closed" - category: "edge" - round: "5.0" - folder_name: "closed-edge" - action: "run" - - os: ubuntu-latest - python-version: "3.12" - division: "closed" - category: "edge" - round: "5.0" - folder_name: "closed-edge" - action: "docker" - - os: ubuntu-latest - python-version: "3.12" - division: "open" - category: "edge" - round: "5.0" - folder_name: "closed-edge" - action: "run" - - os: ubuntu-latest - python-version: "3.12" - division: "open" - category: "edge" - round: "5.0" - folder_name: "closed-edge" - action: "docker" steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -87,13 +58,13 @@ jobs: fi - name: Pull repo where test cases are uploaded run: | - git clone -b submission-generation-examples https://github.com/mlcommons/inference.git submission_generation_examples + git clone -b new-sub-dir https://github.com/anandhu-eng/inference.git submission_generation_examples - name: Run Submission Generation - round-${{ matrix.round }}${{ matrix.folder_name }} ${{ matrix.action }} ${{ matrix.category }} ${{ matrix.division }} run: | if [ "${{ matrix.folder_name }}" == "closed" ]; then description="Test submission - contains closed edge and datacenter" - elif [ "${{ matrix.folder_name }}" == "closed-power" ]; then - description="Test submission - contains closed-power edge and datacenter results" + # elif [ "${{ matrix.folder_name }}" == "closed-power" ]; then + # description="Test submission - contains closed-power edge and datacenter results" fi # Dynamically set the log group to simulate a dynamic step name echo "::group::$description" @@ -104,11 +75,11 @@ jobs: fi if [ -n "${{ inputs.repo-url }}" ]; then - mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags - mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags + mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags else - mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $docker_tags - mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags + mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags fi echo "::endgroup::" diff --git a/automation/script/docker.py b/automation/script/docker.py index 7bf96b8b9..0d4aac13b 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -22,16 +22,7 @@ def dockerfile(self_module, input_params): is_quiet_mode = input_params.get('quiet', False) is_console_output = input_params.get('out') == 'con' - # Step 2: Search for scripts - search_result = self_module.search(input_params.copy()) - if search_result['return'] > 0: - return search_result - - scripts_list = search_result['list'] - if not scripts_list: - return {'return': 1, 'error': 'No scripts were found'} - - # Step 3: Process Dockerfile-related configurations + # Step 2: Process Dockerfile-related configurations env = input_params.get('env', {}) state_data = input_params.get('state', {}) constant_vars = input_params.get('const', {}) @@ -39,190 +30,199 @@ def dockerfile(self_module, input_params): tag_values = input_params.get('tags', '').split(",") variation_tags = [tag[1:] for tag in tag_values if tag.startswith("_")] - # Step 4: Iterate over scripts and generate Dockerfile - for script in sorted(scripts_list, key=lambda x: x.meta.get('alias', '')): - metadata = script.meta - script_directory = script.path - script_tags = metadata.get("tags", []) - script_alias = metadata.get('alias', '') - script_uid = metadata.get('uid', '') - - run_state = { - 'deps': [], - 'fake_deps': [], - 'parent': None, - 'script_id': f"{script_alias},{script_uid}", - 'script_variation_tags': variation_tags - } - docker_settings = metadata.get('docker', {}) - docker_settings_default_env = docker_settings.get('default_env', {}) - for key in docker_settings_default_env: - env.setdefault(key, docker_settings_default_env[key]) - - state_data['docker'] = docker_settings - add_deps_recursive = input_params.get('add_deps_recursive', {}) - - # Update state with metadata and variations - update_state_result = self_module.update_state_from_meta( - metadata, env, state_data, constant_vars, constant_state, - deps=[], - post_deps=[], - prehook_deps=[], - posthook_deps=[], - new_env_keys=[], - new_state_keys=[], - run_state=run_state, - i=input_params - ) - if update_state_result['return'] > 0: - return update_state_result - - update_variations_result = self_module._update_state_from_variations( - input_params, metadata, variation_tags, metadata.get( - 'variations', {}), - env, state_data, constant_vars, constant_state, - deps=[], # Add your dependencies if needed - post_deps=[], # Add post dependencies if needed - prehook_deps=[], # Add prehook dependencies if needed - posthook_deps=[], # Add posthook dependencies if needed - new_env_keys_from_meta=[], # Add keys from meta if needed - new_state_keys_from_meta=[], # Add state keys from meta if needed - add_deps_recursive=add_deps_recursive, - run_state=run_state, - recursion_spaces='' - ) - if update_variations_result['return'] > 0: - return update_variations_result - - # Set Docker-specific configurations - docker_settings = state_data['docker'] - - if is_true(docker_settings.get('pass_docker_to_script', False)): - input_params['docker'] = True - r = self_module.run(input_params) + r = self_module._select_script(input_params) + if r['return'] > 0: + return r + + script = r['script'] + + if not script: + return {'return': 1, + 'error': 'No scripts were found for generating dockerfile'} + + metadata = script.meta + script_directory = script.path + script_tags = metadata.get("tags", []) + script_alias = metadata.get('alias', '') + script_uid = metadata.get('uid', '') + + run_state = { + 'deps': [], + 'fake_deps': [], + 'parent': None, + 'script_id': f"{script_alias},{script_uid}", + 'script_variation_tags': variation_tags + } + + docker_settings = metadata.get('docker', {}) + docker_settings_default_env = docker_settings.get('default_env', {}) + for key in docker_settings_default_env: + env.setdefault(key, docker_settings_default_env[key]) + + state_data['docker'] = docker_settings + add_deps_recursive = input_params.get('add_deps_recursive', {}) + + # Update state with metadata and variations + update_state_result = self_module.update_state_from_meta( + metadata, env, state_data, constant_vars, constant_state, + deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys=[], + new_state_keys=[], + run_state=run_state, + i=input_params + ) + if update_state_result['return'] > 0: + return update_state_result + + update_variations_result = self_module._update_state_from_variations( + input_params, metadata, variation_tags, metadata.get( + 'variations', {}), + env, state_data, constant_vars, constant_state, + deps=[], # Add your dependencies if needed + post_deps=[], # Add post dependencies if needed + prehook_deps=[], # Add prehook dependencies if needed + posthook_deps=[], # Add posthook dependencies if needed + new_env_keys_from_meta=[], # Add keys from meta if needed + new_state_keys_from_meta=[], # Add state keys from meta if needed + add_deps_recursive=add_deps_recursive, + run_state=run_state, + recursion_spaces='' + ) + if update_variations_result['return'] > 0: + return update_variations_result + + # Set Docker-specific configurations + docker_settings = state_data['docker'] + + if is_true(docker_settings.get('pass_docker_to_script', False)): + input_params['docker'] = True + r = self_module.run(input_params) + return r + + if not docker_settings.get('run', True) and not input_params.get( + 'docker_run_override', False): + logger.info("Docker 'run' is set to False in meta.json") + return {'return': 0, 'warning': 'Docker run is set to false in script meta'} + + # Handle build dependencies + show_time = input_params.get('show_time', False) + deps = docker_settings.get('build_deps', []) + if deps: + r = self_module._run_deps( + deps, [], env, {}, {}, {}, add_deps_recursive, '', [], '', False, '', + show_time, ' ', run_state) + if r['return'] > 0: return r - if not docker_settings.get('run', True) and not input_params.get( - 'docker_run_override', False): - logger.info("Docker 'run' is set to False in meta.json") - continue - - # Handle build dependencies - show_time = input_params.get('show_time', False) - deps = docker_settings.get('build_deps', []) - if deps: - r = self_module._run_deps( - deps, [], env, {}, {}, {}, add_deps_recursive, '', [], '', False, '', - show_time, ' ', run_state) - if r['return'] > 0: - return r - - update_state_result = self_module.update_state_from_meta( - metadata, env, state_data, constant_vars, constant_state, - deps=[], - post_deps=[], - prehook_deps=[], - posthook_deps=[], - new_env_keys=[], - new_state_keys=[], - run_state=run_state, - i=input_params - ) - if update_state_result['return'] > 0: - return update_state_result - docker_settings = state_data['docker'] - - # Prune temporary environment variables - run_command = copy.deepcopy(run_command_arc) - for key in list(run_command.get('env', {}).keys()): - if key.startswith("MLC_TMP_"): - del run_command['env'][key] - - # Regenerate script command - regenerate_result = regenerate_script_cmd({ - 'script_uid': script_uid, - 'script_alias': script_alias, - 'run_cmd': run_command, - 'tags': script_tags, - 'fake_run': True, - 'docker_settings': docker_settings, - 'docker_run_cmd_prefix': input_params.get('docker_run_cmd_prefix', docker_settings.get('run_cmd_prefix', '')) - }) - if regenerate_result['return'] > 0: - return regenerate_result - - run_command_string = regenerate_result['run_cmd_string'] - - # Prepare Docker-specific inputs - docker_inputs, dockerfile_path = prepare_docker_inputs( - input_params, docker_settings, script_directory) - - # Handle optional dependencies and comments - if input_params.get('print_deps'): - mlc_input = { - 'action': 'run', 'automation': 'script', 'tags': input_params.get('tags'), - 'print_deps': True, 'quiet': True, 'silent': True, - 'fake_run': True, 'fake_deps': True - } - deps_result = self_module.action_object.access(mlc_input) - if deps_result['return'] > 0: - return deps_result - comments = [ - f"#RUN {dep}" for dep in deps_result['new_state']['print_deps']] - else: - comments = [] - - # Push Docker image if specified - if input_params.get('docker_push_image') in [True, 'True', 'yes']: - env['MLC_DOCKER_PUSH_IMAGE'] = 'yes' - - dockerfile_env = docker_inputs.get('env', {}) - dockerfile_build_env = docker_inputs.get('build_env', {}) - - dockerfile_env['MLC_RUN_STATE_DOCKER'] = True - # Generate Dockerfile - mlc_docker_input = { - 'action': 'run', 'automation': 'script', 'tags': 'build,dockerfile', - 'fake_run_option': " " if docker_inputs.get('real_run') else " --fake_run", - 'comments': comments, 'run_cmd': f"{run_command_string} --quiet", - 'script_tags': input_params.get('tags'), 'env': env, - 'dockerfile_env': dockerfile_env, - 'dockerfile_build_env': dockerfile_build_env, - 'quiet': True, 'real_run': True + update_state_result = self_module.update_state_from_meta( + metadata, env, state_data, constant_vars, constant_state, + deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys=[], + new_state_keys=[], + run_state=run_state, + i=input_params + ) + if update_state_result['return'] > 0: + return update_state_result + docker_settings = state_data['docker'] + + # Prune temporary environment variables + run_command = copy.deepcopy(run_command_arc) + for key in list(run_command.get('env', {}).keys()): + if key.startswith("MLC_TMP_"): + del run_command['env'][key] + + # Regenerate script command + regenerate_result = regenerate_script_cmd({ + 'script_uid': script_uid, + 'script_alias': script_alias, + 'run_cmd': run_command, + 'tags': script_tags, + 'fake_run': True, + 'docker_settings': docker_settings, + 'docker_run_cmd_prefix': input_params.get('docker_run_cmd_prefix', docker_settings.get('run_cmd_prefix', '')) + }) + if regenerate_result['return'] > 0: + return regenerate_result + + run_command_string = regenerate_result['run_cmd_string'] + + # Prepare Docker-specific inputs + docker_inputs, dockerfile_path = prepare_docker_inputs( + input_params, docker_settings, script_directory) + + # Handle optional dependencies and comments + if input_params.get('print_deps'): + mlc_input = { + 'action': 'run', 'automation': 'script', 'tags': input_params.get('tags'), + 'print_deps': True, 'quiet': True, 'silent': True, + 'fake_run': True, 'fake_deps': True } - - docker_v = False + deps_result = self_module.action_object.access(mlc_input) + if deps_result['return'] > 0: + return deps_result + comments = [ + f"#RUN {dep}" for dep in deps_result['new_state']['print_deps']] + else: + comments = [] + + # Push Docker image if specified + if input_params.get('docker_push_image') in [True, 'True', 'yes']: + env['MLC_DOCKER_PUSH_IMAGE'] = 'yes' + + dockerfile_env = docker_inputs.get('env', {}) + dockerfile_build_env = docker_inputs.get('build_env', {}) + + dockerfile_env['MLC_RUN_STATE_DOCKER'] = True + # Generate Dockerfile + mlc_docker_input = { + 'action': 'run', 'automation': 'script', 'tags': 'build,dockerfile', + 'fake_run_option': " " if docker_inputs.get('real_run') else " --fake_run", + 'comments': comments, 'run_cmd': f"{run_command_string} --quiet", + 'script_tags': input_params.get('tags'), 'env': env, + 'dockerfile_env': dockerfile_env, + 'dockerfile_build_env': dockerfile_build_env, + 'quiet': True, 'real_run': True + } + + docker_v = False + docker_s = False + if is_true(input_params.get( + 'docker_v', input_params.get('docker_verbose', False))): + docker_v = True + if is_true(input_params.get( + 'docker_s', input_params.get('docker_silent', False))): + docker_s = True + + if docker_s and docker_v: + logger.warning( + "Both verbose and silent is set to True. Verbose will take precedence.") docker_s = False - if is_true(input_params.get( - 'docker_v', input_params.get('docker_verbose', False))): + + if not docker_s and not docker_v: + if logger.level == logging.DEBUG: docker_v = True - if is_true(input_params.get( - 'docker_s', input_params.get('docker_silent', False))): + elif logger.level == logging.WARNING: docker_s = True - if docker_s and docker_v: - logger.warning( - "Both verbose and silent is set to True. Verbose will take precedence.") - docker_s = False - - if not docker_s and not docker_v: - if logger.level == logging.DEBUG: - docker_v = True - elif logger.level == logging.WARNING: - docker_s = True + if docker_s: + mlc_docker_input['run_cmd'] += ' -s' + elif docker_v: + mlc_docker_input['run_cmd'] += ' -v' - if docker_s: - mlc_docker_input['run_cmd'] += ' -s' - elif docker_v: - mlc_docker_input['run_cmd'] += ' -v' + mlc_docker_input.update(docker_inputs) - mlc_docker_input.update(docker_inputs) + dockerfile_result = self_module.action_object.access(mlc_docker_input) + if dockerfile_result['return'] > 0: + return dockerfile_result - dockerfile_result = self_module.action_object.access(mlc_docker_input) - if dockerfile_result['return'] > 0: - return dockerfile_result - - logger.info(f"Dockerfile generated at {dockerfile_path}") + logger.info(f"Dockerfile generated at {dockerfile_path}") return {'return': 0} @@ -256,13 +256,6 @@ def docker_run(self_module, i): # Save current directory and prepare to search for scripts cur_dir = os.getcwd() - r = self_module.search(i.copy()) - if r['return'] > 0: - return r - - lst = r['list'] - if not lst: - return {'return': 1, 'error': 'No scripts were found'} env['MLC_RUN_STATE_DOCKER'] = False state, const, const_state = i.get( @@ -281,165 +274,173 @@ def docker_run(self_module, i): input_i = copy.deepcopy(i) - # Process each artifact - for artifact in sorted(lst, key=lambda x: x.meta.get('alias', '')): - meta, script_path = artifact.meta, artifact.path - tags, script_alias, script_uid = meta.get( - "tags", []), meta.get( - 'alias', ''), meta.get( - 'uid', '') - - mounts = copy.deepcopy( - i.get( - 'docker_mounts', - [])) # do we need a copy here? - variations = meta.get('variations', {}) - - # take the folder path as well as file path env variables from meta - file_path_env_keys = meta.get('file_path_env_keys', []) - folder_path_env_keys = meta.get('folder_path_env_keys', []) - - docker_settings = meta.get('docker', {}) - docker_settings_default_env = docker_settings.get('default_env', {}) - for key in docker_settings_default_env: - env.setdefault(key, docker_settings_default_env[key]) - - state['docker'] = docker_settings - run_state = { - 'deps': [], 'fake_deps': [], 'parent': None, - 'script_id': f"{script_alias},{script_uid}", - 'script_variation_tags': variation_tags, - 'file_path_env_keys': file_path_env_keys, - 'folder_path_env_keys': folder_path_env_keys - } + r = self_module._select_script(i) + if r['return'] > 0: + return r - # Update state and handle variations - r = self_module.update_state_from_meta(meta, env, state, const, const_state, deps=[], - post_deps=[], - prehook_deps=[], - posthook_deps=[], - new_env_keys=[], - new_state_keys=[], run_state=run_state, i=i) - if r['return'] > 0: - return r + script = r['script'] + + if not script: + return {'return': 1, + 'error': 'No scripts were found for generating dockerfile'} + + meta, script_path = script.meta, script.path + tags, script_alias, script_uid = meta.get( + "tags", []), meta.get( + 'alias', ''), meta.get( + 'uid', '') + + mounts = copy.deepcopy( + i.get( + 'docker_mounts', + [])) # do we need a copy here? + variations = meta.get('variations', {}) + + # take the folder path as well as file path env variables from meta + file_path_env_keys = meta.get('file_path_env_keys', []) + folder_path_env_keys = meta.get('folder_path_env_keys', []) + + docker_settings = meta.get('docker', {}) + docker_settings_default_env = docker_settings.get('default_env', {}) + for key in docker_settings_default_env: + env.setdefault(key, docker_settings_default_env[key]) + + state['docker'] = docker_settings + run_state = { + 'deps': [], 'fake_deps': [], 'parent': None, + 'script_id': f"{script_alias},{script_uid}", + 'script_variation_tags': variation_tags, + 'file_path_env_keys': file_path_env_keys, + 'folder_path_env_keys': folder_path_env_keys + } + + # Update state and handle variations + r = self_module.update_state_from_meta(meta, env, state, const, const_state, deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys=[], + new_state_keys=[], run_state=run_state, i=i) + if r['return'] > 0: + return r - r = self_module._update_state_from_variations( - i, meta, variation_tags, variations, env, state, const, const_state, deps=[], - post_deps=[], - prehook_deps=[], - posthook_deps=[], - new_env_keys_from_meta=[], - new_state_keys_from_meta=[], - add_deps_recursive=add_deps_recursive, run_state=run_state, recursion_spaces='') - if r['return'] > 0: - return r + r = self_module._update_state_from_variations( + i, meta, variation_tags, variations, env, state, const, const_state, deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys_from_meta=[], + new_state_keys_from_meta=[], + add_deps_recursive=add_deps_recursive, run_state=run_state, recursion_spaces='') + if r['return'] > 0: + return r - docker_settings = state['docker'] - - deps = docker_settings.get('deps', []) - if deps: - r = self_module._run_deps( - deps, [], env, {}, {}, {}, add_deps_recursive, '', [], '', False, '', - show_time, ' ', run_state) - if r['return'] > 0: - return r - - # For updating meta from update_meta_if_env - r = self_module.update_state_from_meta( - meta, env, state, const, const_state, deps=[], - post_deps=[], - prehook_deps=[], - posthook_deps=[], - new_env_keys=[], - new_state_keys=[], - run_state=run_state, - i=i) + docker_settings = state['docker'] + + deps = docker_settings.get('deps', []) + if deps: + r = self_module._run_deps( + deps, [], env, {}, {}, {}, add_deps_recursive, '', [], '', False, '', + show_time, ' ', run_state) if r['return'] > 0: return r - # Skip scripts marked as non-runnable - if not docker_settings.get('run', True) and not i.get( - 'docker_run_override', False): - logger.info("docker.run set to False in meta.yaml") - continue + # For updating meta from update_meta_if_env + r = self_module.update_state_from_meta( + meta, env, state, const, const_state, deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys=[], + new_state_keys=[], + run_state=run_state, + i=i) + if r['return'] > 0: + return r - if is_true(docker_settings.get('pass_docker_to_script', False)): - logger.info("Docker 'run' is passed to the run script") - i['docker'] = True - r = self_module.run(i) - return r + # Skip scripts marked as non-runnable + if not docker_settings.get('run', True) and not i.get( + 'docker_run_override', False): + logger.info("docker.run set to False in meta.yaml") + return {'return': 0, 'warning': 'Docker run is set to false in script meta'} - # Regenerate Dockerfile if required - if regenerate_docker_file: - r = dockerfile(self_module, input_i) - if r['return'] > 0: - return r + if is_true(docker_settings.get('pass_docker_to_script', False)): + logger.info("Docker 'run' is passed to the run script") + i['docker'] = True + r = self_module.run(i) + return r - # Ensure Docker is available - r = self_module.action_object.access( - {'action': 'run', 'automation': 'script', 'tags': "get,docker"}) + # Regenerate Dockerfile if required + if regenerate_docker_file: + r = dockerfile(self_module, input_i) if r['return'] > 0: return r - r = self_module._update_env_from_input(env, i) - if r['return'] > 0: - return r + # Ensure Docker is available + r = self_module.action_object.access( + {'action': 'run', 'automation': 'script', 'tags': "get,docker"}) + if r['return'] > 0: + return r - # Prepare Docker-specific inputs - docker_inputs, dockerfile_path = prepare_docker_inputs( - i, docker_settings, script_path, True) - - if docker_inputs is None: - return {'return': 1, 'error': 'Error preparing Docker inputs'} - - docker_input_mapping = docker_settings.get('input_mapping') - - # Update env based on docker_input_mapping if they are in input - if docker_input_mapping and i: - env.update({docker_input_mapping[key]: i[key] - for key in docker_input_mapping if key in i}) - - if docker_inputs.get('user'): - docker_settings['user'] = docker_inputs['user'] - - # Handle environment variable-based mounts - res = process_mounts( - mounts, - env, - docker_settings, - f_run_cmd, - run_state) - if res['return'] > 0: - return res - docker_inputs['mounts'] = res['mounts'] - container_env_string = res['container_env_string'] - - res = update_docker_environment( - docker_settings, env, self_module.host_env_keys, container_env_string) - if res['return'] > 0: - return res - - # Generate the run command - r = regenerate_script_cmd({'script_uid': script_uid, - 'script_alias': script_alias, - 'tags': tags, - 'run_cmd': f_run_cmd}) - if r['return'] > 0: - return r - final_run_cmd = f"""{r['run_cmd_string']} {container_env_string} --docker_run_deps """ - - # Execute the Docker container - mlc_docker_input = { - 'action': 'run', 'target': 'script', 'tags': 'run,docker,container', - 'rebuild': rebuild_docker_image, - 'env': env, 'mounts': mounts, - 'script_tags': i.get('tags'), 'run_cmd': final_run_cmd, - 'quiet': True, 'real_run': True, 'add_deps_recursive': {'build-docker-image': {'dockerfile': dockerfile_path}}, - **docker_inputs - } + r = self_module._update_env_from_input(env, i) + if r['return'] > 0: + return r - r = self_module.action_object.access(mlc_docker_input) - if r['return'] > 0: - return r + # Prepare Docker-specific inputs + docker_inputs, dockerfile_path = prepare_docker_inputs( + i, docker_settings, script_path, True) + + if docker_inputs is None: + return {'return': 1, 'error': 'Error preparing Docker inputs'} + + docker_input_mapping = docker_settings.get('input_mapping') + + # Update env based on docker_input_mapping if they are in input + if docker_input_mapping and i: + env.update({docker_input_mapping[key]: i[key] + for key in docker_input_mapping if key in i}) + + if docker_inputs.get('user'): + docker_settings['user'] = docker_inputs['user'] + + # Handle environment variable-based mounts + res = process_mounts( + mounts, + env, + docker_settings, + f_run_cmd, + run_state) + if res['return'] > 0: + return res + docker_inputs['mounts'] = res['mounts'] + container_env_string = res['container_env_string'] + + res = update_docker_environment( + docker_settings, env, self_module.host_env_keys, container_env_string) + if res['return'] > 0: + return res + + # Generate the run command + r = regenerate_script_cmd({'script_uid': script_uid, + 'script_alias': script_alias, + 'tags': tags, + 'run_cmd': f_run_cmd}) + if r['return'] > 0: + return r + final_run_cmd = f"""{r['run_cmd_string']} {container_env_string} --docker_run_deps """ + + # Execute the Docker container + mlc_docker_input = { + 'action': 'run', 'target': 'script', 'tags': 'run,docker,container', + 'rebuild': rebuild_docker_image, + 'env': env, 'mounts': mounts, + 'script_tags': i.get('tags'), 'run_cmd': final_run_cmd, + 'quiet': True, 'real_run': True, 'add_deps_recursive': {'build-docker-image': {'dockerfile': dockerfile_path}}, + **docker_inputs + } + + r = self_module.action_object.access(mlc_docker_input) + if r['return'] > 0: + return r return {'return': 0} diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index d2f0488e6..79af8f41e 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -145,7 +145,7 @@ def prepare_docker_inputs(input_params, docker_settings, keys += [ "skip_run_cmd", "pre_run_cmds", "run_cmd_prefix", "all_gpus", "num_gpus", "device", "gh_token", "port_maps", "shm_size", "pass_user_id", "pass_user_group", "extra_run_args", "detached", "interactive", - "dt", "it", "use_host_group_id", "use_host_user_id", "keep_detached", "reuse_existing", "use_google_dns" + "dt", "it", "use_host_group_id", "use_host_user_id", "keep_detached", "reuse_existing", "use_google_dns", "privileged" ] # Collect Dockerfile inputs docker_inputs = { diff --git a/automation/script/experiment.py b/automation/script/experiment.py index ac414fa4c..c5b63e5ed 100644 --- a/automation/script/experiment.py +++ b/automation/script/experiment.py @@ -50,48 +50,29 @@ def experiment_run(self_module, i): exp = {} cur_dir = os.getcwd() - r = self_module.search(i.copy()) + + r = self_module._select_script(i) if r['return'] > 0: return r - lst = r['list'] - if not lst: - return {'return': 1, 'error': 'No scripts were found'} - - # Process each artifact - for artifact in sorted(lst, key=lambda x: x.meta.get('alias', '')): - meta, script_path = artifact.meta, artifact.path - tags, script_alias, script_uid = meta.get( - "tags", []), meta.get( - 'alias', ''), meta.get( - 'uid', '') - - # Execute the experiment script - mlc_script_input = { - 'action': 'run', 'target': 'script' - } - if exp: - for key in exp: - ii = {**mlc_script_input, **run_input} - if isinstance(exp[key], list): - for val in exp[key]: - ii[key] = val - r = run_script_and_tag_experiment( - ii, - self_module.action_object, - experiment_action, - tags, - extra_exp_tags, - meta, - skip_state_save, - logger) - if r['return'] > 0: - return r - elif isinstance(exp[key], dict): - return { - 'return': 1, 'error': 'Dictionary inputs are not supported for mlc experiment script'} - else: - ii[key] = exp[key] + script = r['script'] + + meta, script_path = script.meta, script.path + tags, script_alias, script_uid = meta.get( + "tags", []), meta.get( + 'alias', ''), meta.get( + 'uid', '') + + # Execute the experiment script + mlc_script_input = { + 'action': 'run', 'target': 'script' + } + if exp: + for key in exp: + ii = {**mlc_script_input, **run_input} + if isinstance(exp[key], list): + for val in exp[key]: + ii[key] = val r = run_script_and_tag_experiment( ii, self_module.action_object, @@ -103,6 +84,22 @@ def experiment_run(self_module, i): logger) if r['return'] > 0: return r + elif isinstance(exp[key], dict): + return { + 'return': 1, 'error': 'Dictionary inputs are not supported for mlc experiment script'} + else: + ii[key] = exp[key] + r = run_script_and_tag_experiment( + ii, + self_module.action_object, + experiment_action, + tags, + extra_exp_tags, + meta, + skip_state_save, + logger) + if r['return'] > 0: + return r return {'return': 0} diff --git a/automation/script/module.py b/automation/script/module.py index 399196145..bf37ab0eb 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -7,6 +7,7 @@ # Developed by Grigori Fursin and Arjun Suresh for CM and modified for MLCFlow by Arjun Suresh and Anandhu Sooraj # +import re import os import logging @@ -799,7 +800,9 @@ def _run(self, i): run_state['script_repo_git'] = script_item.repo.meta.get( 'git', False) run_state['cache'] = meta.get('cache', False) - run_state['cache_expiration'] = meta.get('cache_expiration', False) + run_state['cache_expiration'] = i.get( + 'cache_expiration', meta.get( + 'cache_expiration', False)) if not recursion: run_state['script_entry_repo_to_report_errors'] = meta.get( @@ -870,13 +873,6 @@ def _run(self, i): prehook_deps = meta.get('prehook_deps', []) posthook_deps = meta.get('posthook_deps', []) - # Store the default_version in run_state -> may be overridden by - # variations - default_version = meta.get( - 'default_version', - '') # not used if version is given - run_state['default_version'] = default_version - # STEP 700: Overwrite env with keys from the script input (to allow user friendly CLI) # IT HAS THE PRIORITY OVER meta['default_env'] and meta['env'] but not over the meta from versions/variations # (env OVERWRITE - user enforces it from CLI) @@ -1469,7 +1465,7 @@ def _run(self, i): # Update default version meta if version is not set if version == '': - default_version = run_state.get('default_version', '') + default_version = meta.get('default_version', '') if default_version != '': version = default_version @@ -2343,244 +2339,310 @@ def _dump_version_info_for_script( return {'return': 0} - ########################################################################## - def _update_state_from_variations(self, i, meta, variation_tags, variations, env, state, const, const_state, deps, post_deps, prehook_deps, - posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, add_deps_recursive, run_state, recursion_spaces): - - logger = self.action_object.logger - # Save current explicit variations + def _update_state_from_variations( + self, i, meta, variation_tags, variations, env, state, + const, const_state, deps, post_deps, prehook_deps, + posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, + add_deps_recursive, run_state, recursion_spaces + ): import copy - explicit_variation_tags = copy.deepcopy(variation_tags) + logger = self.action_object.logger + explicit_variation_tags = copy.deepcopy(variation_tags) if add_deps_recursive is None: add_deps_recursive = {} - - # Calculate space required_disk_space = {} - - # Check if warning warnings = [] - # variation_tags get appended by any aliases + # 1️⃣ Expand and validate variation tags r = self._get_variations_with_aliases(variation_tags, variations) if r['return'] > 0: return r variation_tags = r['variation_tags'] excluded_variation_tags = r['excluded_variation_tags'] - # Get a dictionary of variation groups r = self._get_variation_groups(variations) if r['return'] > 0: return r - variation_groups = r['variation_groups'] - run_state['variation_groups'] = variation_groups - # Add variation(s) if specified in the "tags" input prefixed by _ - # If there is only 1 default variation, then just use it or - # substitute from CMD - default_variation = meta.get('default_variation', '') - if default_variation and default_variation not in variations: - return {'return': 1, 'error': 'Default variation "{}" is not in the list of variations: "{}" '.format( - default_variation, variations.keys())} + return { + 'return': 1, 'error': f'Default variation "{default_variation}" not in {list(variations.keys())}'} - if len(variation_tags) == 0: - if default_variation != '' and default_variation not in excluded_variation_tags: - variation_tags = [default_variation] + if not variation_tags and default_variation and default_variation not in excluded_variation_tags: + variation_tags = [default_variation] r = self._update_variation_tags_from_variations( - variation_tags, variations, variation_groups, excluded_variation_tags) + variation_tags, variation_groups, excluded_variation_tags, variations) if r['return'] > 0: return r - # variation_tags get appended by any default on variation in groups + # process group defaults r = self._process_variation_tags_in_groups( variation_tags, variation_groups, excluded_variation_tags, variations) if r['return'] > 0: return r if variation_tags != r['variation_tags']: variation_tags = r['variation_tags'] - # we need to again process variation tags if any new default # variation is added r = self._update_variation_tags_from_variations( - variation_tags, variations, variation_groups, excluded_variation_tags) + variation_tags, variation_groups, excluded_variation_tags, variations) if r['return'] > 0: return r - valid_variation_combinations = meta.get( - 'valid_variation_combinations', []) - if valid_variation_combinations: - if not any(all(t in variation_tags for t in s) - for s in valid_variation_combinations): - return {'return': 1, 'error': 'Invalid variation combination "{}" prepared. Valid combinations: "{}" '.format( - variation_tags, valid_variation_combinations)} - - invalid_variation_combinations = meta.get( - 'invalid_variation_combinations', []) - if invalid_variation_combinations: - if any(all(t in variation_tags for t in s) - for s in invalid_variation_combinations): - return {'return': 1, 'error': 'Invalid variation combination "{}" prepared. Invalid combinations: "{}" '.format( - variation_tags, invalid_variation_combinations)} + # Validate combinations + r = self._validate_variations(meta, variation_tags) + if r['return'] > 0: + return r variation_tags_string = '' - if len(variation_tags) > 0: - for t in variation_tags: - if variation_tags_string != '': - variation_tags_string += ',' - - x = '_' + t - variation_tags_string += x + if variation_tags: + variation_tags_string = ','.join(['_' + t for t in variation_tags]) logger.debug( - recursion_spaces + - ' Prepared variations: {}'.format(variation_tags_string)) + f"{recursion_spaces}Prepared variations: {variation_tags_string}") - # Update env and other keys if variations - if len(variation_tags) > 0: + # 2️⃣ Apply individual variations for variation_tag in variation_tags: - if variation_tag.startswith('~'): - # ignore such tag (needed for caching only to differentiate - # variations) - continue - - if variation_tag.startswith('-'): - # ignore such tag (needed for caching only to eliminate - # variations) - continue - - variation_tag_dynamic_suffix = None - if variation_tag not in variations: - if '.' in variation_tag and variation_tag[-1] != '.': - variation_tag_dynamic_suffix = variation_tag[variation_tag.index( - ".") + 1:] - if not variation_tag_dynamic_suffix: - return {'return': 1, 'error': 'tag {} is not in variations {}'.format( - variation_tag, variations.keys())} - variation_tag = self._get_name_for_dynamic_variation_tag( - variation_tag) - if variation_tag not in variations: - return {'return': 1, 'error': 'tag {} is not in variations {}'.format( - variation_tag, variations.keys())} - - variation_meta = variations[variation_tag] - if variation_tag_dynamic_suffix: - variation_meta = copy.deepcopy(variation_meta) - self._update_variation_meta_with_dynamic_suffix( - variation_meta, variation_tag_dynamic_suffix) - - r = update_state_from_meta( - variation_meta, - env, - state, - const, - const_state, - deps, - post_deps, - prehook_deps, - posthook_deps, - new_env_keys_from_meta, - new_state_keys_from_meta, - run_state, - i) + r = self._apply_single_variation( + variation_tag, variations, env, state, const, const_state, + deps, post_deps, prehook_deps, posthook_deps, + new_env_keys_from_meta, new_state_keys_from_meta, + run_state, i, meta, required_disk_space, warnings, add_deps_recursive + ) if r['return'] > 0: return r - if variation_meta.get('script_name', '') != '': - meta['script_name'] = variation_meta['script_name'] + # 3️⃣ Apply combined variations + r = self._apply_combined_variations( + variations, variation_tags, env, state, const, const_state, + deps, post_deps, prehook_deps, posthook_deps, + new_env_keys_from_meta, new_state_keys_from_meta, + run_state, i, meta, required_disk_space, warnings, add_deps_recursive + ) + if r['return'] > 0: + return r - if variation_meta.get('default_version', '') != '': - run_state['default_version'] = variation_meta['default_version'] + # Processing them again using updated deps for add_deps_recursive + r = update_adr_from_meta( + deps, + post_deps, + prehook_deps, + posthook_deps, + add_deps_recursive, + env) + if r['return'] > 0: + return r - if variation_meta.get( - 'required_disk_space', 0) > 0 and variation_tag not in required_disk_space: - required_disk_space[variation_tag] = variation_meta['required_disk_space'] + # Done + return { + 'return': 0, + 'variation_tags_string': variation_tags_string, + 'explicit_variation_tags': explicit_variation_tags, + 'warnings': warnings, + 'required_disk_space': required_disk_space, + 'variation_tags': variation_tags + } - if variation_meta.get('warning', '') != '': - x = variation_meta['warning'] - if x not in warnings: - warnings.append() + def _apply_variation_meta(self, variation_key, variation_meta, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, + new_env_keys_from_meta, new_state_keys_from_meta, run_state, i, meta, required_disk_space, warnings, add_deps_recursive): + r = update_state_from_meta( + variation_meta, + env, + state, + const, + const_state, + deps, + post_deps, + prehook_deps, + posthook_deps, + new_env_keys_from_meta, + new_state_keys_from_meta, + run_state, + i) + if r['return'] > 0: + return r - adr = get_adr(variation_meta) - if adr: - self._merge_dicts_with_tags(add_deps_recursive, adr) + adr = get_adr(variation_meta) + if adr: + self._merge_dicts_with_tags(add_deps_recursive, adr) - combined_variations = [t for t in variations if ',' in t] + # Merge selected meta keys + for k in ('script_name', 'default_version'): + if k in variation_meta: + meta[k] = variation_meta[k] - combined_variations.sort(key=lambda x: x.count(',')) - ''' By sorting based on the number of variations users can safely override - env and state in a larger combined variation - ''' + # Track required disk space per variation key + rds = variation_meta.get('required_disk_space', 0) + if rds > 0: + required_disk_space[variation_key] = rds - for combined_variation in combined_variations: - v = combined_variation.split(",") - all_present = set(v).issubset(set(variation_tags)) - if all_present: + # Collect unique warnings + x = variation_meta.get('warning') + if x and x not in warnings: + warnings.append(x) - combined_variation_meta = variations[combined_variation] + return {'return': 0} - r = update_state_from_meta( - combined_variation_meta, - env, - state, - const, - const_state, - deps, - post_deps, - prehook_deps, - posthook_deps, - new_env_keys_from_meta, - new_state_keys_from_meta, - run_state, - i) - if r['return'] > 0: - return r + def _resolve_dynamic_tag(self, tag, variations): + # Returns (base_tag, suffix) for a possibly dynamic tag. # If not + # dynamic or cannot be resolved, returns (tag, None). + + if '.' in tag and tag[-1] != '.': + suffix = tag.split('.', 1)[1] + base = self._get_name_for_dynamic_variation_tag(tag) + + # Only accept if base is a known variation key; otherwise treat as + # non-dynamic + if base in variations: + return base, suffix + return tag, None + + def _apply_single_variation( + self, variation_tag, variations, env, state, const, const_state, + deps, post_deps, prehook_deps, posthook_deps, + new_env_keys_from_meta, new_state_keys_from_meta, + run_state, i, meta, required_disk_space, warnings, add_deps_recursive + ): + import copy + if variation_tag.startswith(('~', '-')): + return {'return': 0} - adr = get_adr(combined_variation_meta) - if adr: - self._merge_dicts_with_tags( - add_deps_recursive, adr) + variation_tag_dynamic_suffix = None + if variation_tag not in variations: + if '.' in variation_tag and variation_tag[-1] != '.': + variation_tag_dynamic_suffix = variation_tag.split('.', 1)[1] + variation_tag = self._get_name_for_dynamic_variation_tag( + variation_tag) + if variation_tag not in variations: + return { + 'return': 1, + 'error': f'Tag {variation_tag} not in variations {list(variations.keys())}' + } - if combined_variation_meta.get( - 'script_name', '') != '': - meta['script_name'] = combined_variation_meta['script_name'] + variation_meta = variations[variation_tag] + if variation_tag_dynamic_suffix: + variation_meta = copy.deepcopy(variation_meta) + self._update_variation_meta_with_dynamic_suffix( + variation_meta, variation_tag_dynamic_suffix + ) + + return self._apply_variation_meta( + variation_key=variation_tag, + variation_meta=variation_meta, + env=env, state=state, const=const, const_state=const_state, + deps=deps, post_deps=post_deps, prehook_deps=prehook_deps, posthook_deps=posthook_deps, + new_env_keys_from_meta=new_env_keys_from_meta, new_state_keys_from_meta=new_state_keys_from_meta, + run_state=run_state, i=i, + meta=meta, + required_disk_space=required_disk_space, + warnings=warnings, + add_deps_recursive=add_deps_recursive + ) + + def _is_dynamic_placeholder(self, comp): + return isinstance(comp, str) and comp.endswith('.#') + + def _dynamic_base(self, comp): # Assumes comp endswith ".#" + return comp[:-2] + + def _apply_combined_variations( + self, variations, variation_tags, env, state, const, const_state, + deps, post_deps, prehook_deps, posthook_deps, + new_env_keys_from_meta, new_state_keys_from_meta, + run_state, i, meta, required_disk_space, warnings, add_deps_recursive + ): - if variation_meta.get('default_version', '') != '': - run_state['default_version'] = variation_meta['default_version'] + # Only consider string keys that are combined (contain a comma) + combined_variations = [ + t for t in variations if isinstance( + t, str) and ',' in t] + # Ensure we apply more specific (more components) first + combined_variations.sort(key=lambda x: x.count(','), reverse=False) + + # Pre-resolve selected tags into base->(original_tag, suffix) so we can match combined keys by base + # If multiple selected tags map to the same base, keep the first + # occurrence + selected_by_base = {} + for sel in variation_tags: + base, suffix = self._resolve_dynamic_tag(sel, variations) + if base not in selected_by_base: + selected_by_base[base] = (sel, suffix) + + for combined_key in combined_variations: + components = combined_key.split(',') + + # Split into dynamic placeholders and static components + dyn_components = [ + c for c in components if self._is_dynamic_placeholder(c)] + static_components = [ + c for c in components if not self._is_dynamic_placeholder(c)] + + # For static components, require presence via relaxed_subset + # (so a selected "blas.3" can satisfy static "blas" if relaxed_subset allows it) + static_ok = set(static_components).issubset(set(variation_tags)) + if not static_ok: + continue - if combined_variation_meta.get( - 'required_disk_space', 0) > 0 and combined_variation not in required_disk_space: - required_disk_space[combined_variation] = combined_variation_meta['required_disk_space'] + # For dynamic placeholders, require the base to be present in + # selected tags + dyn_bases = [self._dynamic_base(c) + '.#' for c in dyn_components] + if not all(b in selected_by_base for b in dyn_bases): + continue - if combined_variation_meta.get('warning', '') != '': - x = combined_variation_meta['warning'] - if x not in warnings: - warnings.append(x) + # Enforce at most one dynamic component among these bases + dynamic_infos = [(b, selected_by_base[b][1]) for b in dyn_bases] + dynamic_infos = [(b, sfx) for b, sfx in dynamic_infos if sfx] + if len(dynamic_infos) > 1: + # Skip if more than one dynamic suffix is present + continue - # Processing them again using updated deps for add_deps_recursive - r = update_adr_from_meta( - deps, - post_deps, - prehook_deps, - posthook_deps, - add_deps_recursive, - env) + combined_meta = variations[combined_key] + + # If exactly one dynamic suffix among components, apply it to a + # copy of the combined meta + if len(dynamic_infos) == 1: + _, dyn_suffix = dynamic_infos[0] + import copy + combined_meta = copy.deepcopy(combined_meta) + self._update_variation_meta_with_dynamic_suffix( + combined_meta, dyn_suffix) + + r = self._apply_variation_meta( + variation_key=combined_key, + variation_meta=combined_meta, + env=env, state=state, const=const, const_state=const_state, + deps=deps, post_deps=post_deps, prehook_deps=prehook_deps, posthook_deps=posthook_deps, + new_env_keys_from_meta=new_env_keys_from_meta, new_state_keys_from_meta=new_state_keys_from_meta, + run_state=run_state, i=i, + meta=meta, + required_disk_space=required_disk_space, + warnings=warnings, + add_deps_recursive=add_deps_recursive + ) if r['return'] > 0: return r - if len(required_disk_space) > 0: - required_disk_space_sum_mb = sum( - list(required_disk_space.values())) + return {'return': 0} + + def _validate_variations(self, meta, variation_tags): + valid = meta.get('valid_variation_combinations', []) + if valid: + if not any(all(t in variation_tags for t in s) for s in valid): + return { + 'return': 1, 'error': f'Invalid combination {variation_tags}, valid: {valid}'} - warnings.append( - 'Required disk space: {} MB'.format(required_disk_space_sum_mb)) + invalid = meta.get('invalid_variation_combinations', []) + if invalid: + if any(all(t in variation_tags for t in s) for s in invalid): + return { + 'return': 1, 'error': f'Combination {variation_tags} is explicitly invalid: {invalid}'} - return {'return': 0, 'variation_tags_string': variation_tags_string, - 'explicit_variation_tags': explicit_variation_tags, 'warnings': warnings} + return {'return': 0} def _add_base_variations( self, @@ -2640,7 +2702,7 @@ def _add_base_variations( ########################################################################## def _update_variation_tags_from_variations( - self, variation_tags, variations, variation_groups, excluded_variation_tags): + self, variation_tags, variation_groups, excluded_variation_tags, variations): import copy tmp_variation_tags_static = copy.deepcopy(variation_tags) @@ -2711,7 +2773,7 @@ def _update_variation_tags_from_variations( if tmp_combined_variations[combined_variation]: continue v = combined_variation.split(",") - all_present = set(v).issubset(set(variation_tags)) + all_present = relaxed_subset(v, variation_tags) if all_present: combined_variation_meta = variations[combined_variation] tmp_combined_variations[combined_variation] = True @@ -2755,6 +2817,7 @@ def _update_variation_tags_from_variations( return {'return': 0} ########################################################################## + def _get_variation_tags_from_default_variations( self, variation_meta, variations, variation_groups, tmp_variation_tags_static, excluded_variation_tags): # default_variations dictionary specifies the default_variation for @@ -4590,6 +4653,11 @@ def experiment(self, i): from script.experiment import experiment_run return experiment_run(self, i) + ############################################################ + def remote_run(self, i): + from script.remote_run import remote_run + return remote_run(self, i) + ########################################################################## def _available_variations(self, i): @@ -4649,6 +4717,71 @@ def clean_some_tmp_files(self, i): def _get_script_name(self, env, path, filename="run"): return get_script_name(env, path, filename) + def _select_script(self, i): + r = self.search(i.copy()) + if r['return'] > 0: + return r + + lst = r['list'] + if not lst: + return {'return': 1, 'error': 'No scripts were found'} + + # Sort scripts by alias for consistent display + sorted_list = sorted(lst, key=lambda x: x.meta.get('alias', '')) + + # If quiet mode is off, prompt the user + if not i.get('quiet', False) and len(sorted_list) > 1: + print("\nAvailable scripts:") + for idx, artifact in enumerate(sorted_list, 1): + meta = artifact.meta + alias = meta.get('alias', '') + uid = meta.get('uid', '') + tags = ', '.join(meta.get('tags', [])) + print(f"{idx}. {alias or uid} [{tags}]") + + try: + choice = int(input("\nSelect a script by number: ")) - 1 + except ValueError: + return {'return': 1, 'error': 'Invalid selection'} + + if choice < 0 or choice >= len(sorted_list): + return {'return': 1, 'error': 'Selection out of range'} + + selected_artifact = sorted_list[choice] + else: + # Quiet mode: select the first script + selected_artifact = sorted_list[0] + + return { + 'return': 0, + 'script': selected_artifact + } + +########################################################################## + + +def relaxed_subset(v, variation_tags): + """ + Returns True if every element in v matches at least one element + in variation_tags, allowing patterns like 'build.#' to match + anything that starts with 'build.'. + """ + for item in v: + matched = False + for tag in variation_tags: + if item == tag: + matched = True + break + # Relaxed match: 'build.#' -> matches 'build.' + anything + if item.endswith(".#"): + prefix = item[:-2] + if tag.startswith(prefix + "."): + matched = True + break + if not matched: + return False + return True + def get_version_tag_from_version(version, cached_tags): tags_to_add = [] diff --git a/automation/script/remote_run.py b/automation/script/remote_run.py new file mode 100644 index 000000000..f0ea97aac --- /dev/null +++ b/automation/script/remote_run.py @@ -0,0 +1,88 @@ +from collections import defaultdict +import os +import mlc.utils as utils +from mlc import utils +from utils import * +import logging +from pathlib import PureWindowsPath, PurePosixPath +import time +import copy +from datetime import datetime + + +def remote_run(self_module, i): + """ + Remote run of MLC scripts. + + Args: + self_module: Reference to the current module for internal calls. + i: Dictionary containing input parameters for the experiment execution. + + Returns: + Dictionary with the result of the operation. Keys: + - 'return': 0 on success, >0 on error. + - 'error': Error message (if any). + """ + + # Extract and handle basic inputs + quiet = i.get('quiet', False) + show_time = i.get('show_time', False) + logger = self_module.logger + env = i.get('env', {}) + remote_host = i.get('remote_host', 'localhost') + remote_port = i.get('remote_port', 22) + + prune_result = prune_input( + {'input': i, 'extra_keys_starts_with': ['remote_']}) + if prune_result['return'] > 0: + return prune_result + + run_input = prune_result['new_input'] + mlc_run_cmd = run_input['mlc_run_cmd'] + + cur_dir = os.getcwd() + + r = self_module._select_script(i) + if r['return'] > 0: + return r + + script = r['script'] + + meta, script_path = script.meta, script.path + tags, script_alias, script_uid = meta.get( + "tags", []), meta.get( + 'alias', ''), meta.get( + 'uid', '') + + # Execute the experiment script + mlc_script_input = { + 'action': 'run', 'target': 'script' + } + + run_cmds = [] + remote_mlc_python_venv = i.get('remote_python_venv', 'mlcflow') + run_cmds.append(f". {remote_mlc_python_venv}/bin/activate") + if i.get('remote_pull_mlc_repos', False): + run_cmds.append("mlc pull repo") + + script_run_cmd = " ".join(mlc_run_cmd.split(" ")[3:]) + run_cmds.append(f"mlcr {script_run_cmd}") + + remote_inputs = {} + for key in ["host", "port", "user", "client_refresh", + "password", "skip_host_verify", "ssh_key_file"]: + if i.get(f"remote_{key}"): + remote_inputs[key] = i[f"remote_{key}"] + + # Execute the remote command + mlc_remote_input = { + 'action': 'run', 'target': 'script', 'tags': 'remote,run,cmds,ssh', + 'script_tags': i.get('tags'), 'run_cmds': run_cmds, + **remote_inputs + } + + r = self_module.action_object.access(mlc_remote_input) + if r['return'] > 0: + return r + + return {'return': 0} diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index 3fdff095d..85ab46001 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -372,7 +372,7 @@ deps: - retinanet ## GPT-J - - tags: get,ml-model,large-language-model,gptj + - tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader names: - ml-model - gptj-model @@ -450,7 +450,7 @@ deps: - "yes" ## mixtral-8x7b - - tags: get,ml-model,mixtral + - tags: get,ml-model,mixtral,_r2-downloader,_mlc names: - ml-model - mixtral-model @@ -501,15 +501,17 @@ deps: - "on" ## RGAT - - tags: get,ml-model,rgat + - tags: get,ml-model,rgat,_r2-downloader,_mlcommons names: - rgat-model enable_if_env: MLC_MODEL: - rgat skip_if_env: - RGAT_CHECKPOINT_PATH: - - "on" + MLC_USE_MODEL_FROM_HOST: + - "yes" + MLC_RUN_STATE_DOCKER: + - "yes" ## LLAMA3_1-405B - tags: get,ml-model,llama3 @@ -540,7 +542,7 @@ deps: - "yes" ## pointpainting - - tags: get,ml-model,pointpainting + - tags: get,ml-model,pointpainting,_r2-downloader,_mlc names: - pointpillars-model enable_if_env: @@ -690,7 +692,7 @@ deps: - "yes" ## waymo for pointpillats - - tags: get,dataset,waymo + - tags: get,dataset,waymo,_r2-downloader,_mlc enable_if_env: MLC_MODEL: - pointpainting @@ -701,7 +703,7 @@ deps: - waymo-dataset ## llama3_1 dataset - - tags: get,dataset,mlperf,inference,llama3,_validation + - tags: get,dataset,mlperf,inference,llama3,_validation,_r2-downloader names: - llama3_1-dataset - llama3-dataset @@ -1462,13 +1464,6 @@ variations: - tags: get,generic-python-lib,_package.evaluate - tags: get,generic-python-lib,_package.absl-py - tags: get,generic-python-lib,_package.rouge-score - - tags: get,ml-model,llama3,_hf,_meta-llama/Llama-3.1-8B-Instruct - skip_if_env: - MLC_RUN_STATE_DOCKER: - - "yes" - names: - - llama3_1-8b - - llama3-8b llama3_1-8b: base: @@ -1477,7 +1472,7 @@ variations: MLC_MODEL: llama3_1-8b deps: ## CNNDM for Llama3 8B model - datacenter - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3 + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-datacenter skip_if_env: @@ -1491,7 +1486,7 @@ variations: MLC_MODEL: llama3_1-8b-edge deps: ## CNNDM for Llama3 8B model - edge - - tags: get,dataset,cnndm,_validation,_edge,_llama3 + - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-edge skip_if_env: diff --git a/script/app-mlperf-inference-nvidia/meta.yaml b/script/app-mlperf-inference-nvidia/meta.yaml index c2f9b3bdc..09af92c8b 100644 --- a/script/app-mlperf-inference-nvidia/meta.yaml +++ b/script/app-mlperf-inference-nvidia/meta.yaml @@ -306,7 +306,7 @@ prehook_deps: - preprocess_data names: - gptj-model - tags: get,ml-model,gptj,_pytorch,_rclone + tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader # Download model for sdxl - enable_if_env: @@ -318,7 +318,7 @@ prehook_deps: - stable-diffusion-xl - sdxl-model - ml-model - tags: get,ml-model,sdxl,_fp16,_rclone + tags: get,ml-model,sdxl,_fp16,_r2-downloader skip_if_env: MLC_RUN_STATE_DOCKER: - 'yes' diff --git a/script/app-mlperf-inference-redhat/meta.yaml b/script/app-mlperf-inference-redhat/meta.yaml index 452e8094e..d02b964bd 100644 --- a/script/app-mlperf-inference-redhat/meta.yaml +++ b/script/app-mlperf-inference-redhat/meta.yaml @@ -221,7 +221,7 @@ variations: gptj_: deps: - - tags: get,ml-model,gptj + - tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader names: - gptj-model - tags: get,dataset,cnndm,_validation diff --git a/script/app-mlperf-inference/customize.py b/script/app-mlperf-inference/customize.py index 62c05a287..fb346e496 100644 --- a/script/app-mlperf-inference/customize.py +++ b/script/app-mlperf-inference/customize.py @@ -504,12 +504,21 @@ def postprocess(i): COMPLIANCE_DIR = output_dir OUTPUT_DIR = os.path.dirname(COMPLIANCE_DIR) + # For backward compatibility with inference commit + # d9efabf829526234629a4dfe5cc34a36cb81417c SCRIPT_PATH = os.path.join( env['MLC_MLPERF_INFERENCE_SOURCE'], "compliance", "nvidia", test, "run_verification.py") + if not os.path.exists(SCRIPT_PATH): + SCRIPT_PATH = os.path.join( + env['MLC_MLPERF_INFERENCE_SOURCE'], + "compliance", + test, + "run_verification.py") + if test == "TEST06": cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {q}{SCRIPT_PATH}{q} -c {q}{COMPLIANCE_DIR}{q} -o {q}{OUTPUT_DIR}{q} --scenario {scenario} --dtype int32""" else: @@ -523,8 +532,14 @@ def postprocess(i): run_script_input = i['run_script_input'] automation = i['automation'] + # For backward compatibility with inference commit + # d9efabf829526234629a4dfe5cc34a36cb81417c SCRIPT_PATH = os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "compliance", "nvidia", test, "create_accuracy_baseline.sh") + if not os.path.exists(SCRIPT_PATH): + SCRIPT_PATH = os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "compliance", test, + "create_accuracy_baseline.sh") + TEST01_DIR = os.path.join(OUTPUT_DIR, "TEST01") OUTPUT_DIR = os.path.join(OUTPUT_DIR, "TEST01", "accuracy") if not os.path.exists(OUTPUT_DIR): diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index 7d5dd3fbf..feb8b7452 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -317,7 +317,7 @@ variations: all-models,nvidia-original: docker: deps: - - tags: get,ml-model,gptj,raw + - tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader skip_if_env: MLC_MLPERF_NVIDIA_SKIP_GPTJ: - "yes" @@ -704,7 +704,7 @@ variations: implementation docker: interactive: True - extra_run_args: ' --privileged' + privileged: True mounts: - "${{ MLC_MLPERF_INFERENCE_INTEL_GPTJ_INT8_MODEL_PATH }}:${{ MLC_MLPERF_INFERENCE_INTEL_GPTJ_INT8_MODEL_PATH }}" - "${{ GPTJ_CHECKPOINT_PATH }}:${{ GPTJ_CHECKPOINT_PATH }}" @@ -748,7 +748,7 @@ variations: implementation docker: interactive: True - extra_run_args: ' --privileged' + privileged: True mounts: - "${{ LLAMA2_CHECKPOINT_PATH }}:${{ LLAMA2_CHECKPOINT_PATH }}" - "${{ GPTJ_CHECKPOINT_PATH }}:${{ GPTJ_CHECKPOINT_PATH }}" @@ -1018,6 +1018,12 @@ variations: names: - igbh-original - igbh-dataset + - tags: get,ml-model,rgat,_r2-downloader,_mlcommons + enable_if_env: + MLC_USE_ML_MODEL_FROM_HOST: + - 'yes' + names: + - rgat-model pointpainting: group: @@ -1030,13 +1036,13 @@ variations: pointpainting docker: deps: - - tags: get,dataset,waymo + - tags: get,dataset,waymo,_r2-downloader,_mlc enable_if_env: MLC_USE_DATASET_FROM_HOST: - 'yes' names: - waymo-dataset - - tags: get,ml-model,pointpainting + - tags: get,ml-model,pointpainting,_r2-downloader,_mlc enable_if_env: MLC_USE_ML_MODEL_FROM_HOST: - 'yes' @@ -1225,6 +1231,13 @@ variations: names: - llama3_1-405b - llama3-405b + - tags: get,dataset,mlperf,inference,llama3,_validation,_r2-downloader + enable_if_env: + MLC_USE_DATASET_FROM_HOST: + - 'yes' + names: + - llama3_1-dataset + - llama3-dataset deepseek-r1: group: @@ -1303,7 +1316,7 @@ variations: - 'yes' MLC_USE_MODEL_FROM_HOST: - 'yes' - tags: get,ml-model,sdxl,_fp16,_rclone + tags: get,ml-model,sdxl,_fp16,_r2-downloader sdxl,reference,float16: docker: @@ -1314,7 +1327,7 @@ variations: - 'yes' MLC_USE_MODEL_FROM_HOST: - 'yes' - tags: get,ml-model,sdxl,_fp16,_rclone + tags: get,ml-model,sdxl,_fp16,_r2-downloader sdxl,reference,bfloat16: docker: @@ -1325,7 +1338,7 @@ variations: - 'yes' MLC_USE_MODEL_FROM_HOST: - 'yes' - tags: get,ml-model,sdxl,_fp16,_rclone + tags: get,ml-model,sdxl,_fp16,__r2-downloader sdxl,reference,float32: docker: @@ -1336,7 +1349,7 @@ variations: - 'yes' MLC_USE_MODEL_FROM_HOST: - 'yes' - tags: get,ml-model,sdxl,_fp32,_rclone + tags: get,ml-model,sdxl,_fp32,__r2-downloader llama2-70b_: default_env: @@ -1450,7 +1463,7 @@ variations: docker: image_name: mlperf-inference-mlcommons-python-implementation-mixtral-8x7b deps: - - tags: get,ml-model,mixtral + - tags: get,ml-model,mixtral,_r2-downloader,_mlc names: - ml-model - mixtral-model @@ -1527,7 +1540,7 @@ variations: gptj_: docker: deps: - - tags: get,ml-model,gptj,raw + - tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader env: MLC_MLPERF_MODEL_EQUAL_ISSUE_MODE: 'yes' posthook_deps: diff --git a/script/build-mlperf-inference-server-nvidia/customize.py b/script/build-mlperf-inference-server-nvidia/customize.py index f6a2f399f..3966021ac 100644 --- a/script/build-mlperf-inference-server-nvidia/customize.py +++ b/script/build-mlperf-inference-server-nvidia/customize.py @@ -19,8 +19,8 @@ def preprocess(i): env['+LIBRARY_PATH'].append(os.path.join( env['MLC_TENSORRT_INSTALL_PATH'], "lib")) + hpcx_paths = [] if is_true(env.get('BUILD_TRTLLM')): - hpcx_paths = [] if os.path.exists("/opt/hpcx/ucx/lib"): hpcx_paths.append("/opt/hpcx/ucx/lib") if os.path.exists("/opt/hpcx/ucc/lib"): diff --git a/script/download-and-extract/README.md b/script/download-and-extract/README.md new file mode 100644 index 000000000..09a080c89 --- /dev/null +++ b/script/download-and-extract/README.md @@ -0,0 +1,90 @@ +# README for download-and-extract +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr download-and-extract file +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--download_path` | | | `` | +| `--extra_folder` | | | `` | +| `--extract_path` | | | `` | +| `--from` | | | `` | +| `--local_path` | Alias for from | | `` | +| `--store` | Alias for download_path | | `` | +| `--to` | Alias for extract_path | | `` | +| `--url` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Download-tool + +- `curl` +- `gdown` +- `mlcutil` (alias: cmutil) (default) +- `r2-downloader` +- `rclone` +- `torrent` +- `wget` + +### Keep + +- `keep` (default) +- `remove-extracted` + +### Ungrouped + +- `extract` +- `extract-to-download-dir` +- `url.#` _(# can be substituted dynamically)_ diff --git a/script/download-and-extract/customize.py b/script/download-and-extract/customize.py index 4a11b4fd5..bc3d2a6ad 100644 --- a/script/download-and-extract/customize.py +++ b/script/download-and-extract/customize.py @@ -48,6 +48,9 @@ def preprocess(i): env.get('MLC_DAE_FINAL_ENV_NAME', '') != ''): env['MLC_EXTRACT_FINAL_ENV_NAME'] = env['MLC_DAE_FINAL_ENV_NAME'] + if is_true(env.get('MLC_DAE_EXTRACT_TO_DOWNLOADED')): + env['MLC_OUTDIRNAME'] = os.getcwd() + return {'return': 0} diff --git a/script/download-and-extract/meta.yaml b/script/download-and-extract/meta.yaml index 85fcd25ac..e031655d5 100644 --- a/script/download-and-extract/meta.yaml +++ b/script/download-and-extract/meta.yaml @@ -78,6 +78,9 @@ variations: extract: env: MLC_DAE_EXTRACT_DOWNLOADED: 'yes' + extract-to-download-dir: + env: + MLC_DAE_EXTRACT_TO_DOWNLOADED: 'yes' gdown: add_deps: download-script: diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 8ca22b596..c53bcf853 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -309,16 +309,11 @@ def generate_submission(env, state, inp, submission_division, logger): sub_res = new_res submission_path = os.path.join(path_submission, "results", sub_res) - measurement_path = os.path.join( - path_submission, "measurements", sub_res) - compliance_path = os.path.join(path_submission, "compliance", sub_res) system_path = os.path.join(path_submission, "systems") submission_system_path = system_path if not os.path.isdir(submission_path): os.makedirs(submission_path) - if not os.path.isdir(measurement_path): - os.makedirs(measurement_path) if not os.path.isdir(submission_system_path): os.makedirs(submission_system_path) system_file = os.path.join(submission_system_path, sub_res + ".json") @@ -342,8 +337,6 @@ def generate_submission(env, state, inp, submission_division, logger): platform_info_file = None result_model_path = os.path.join(result_path, model) submission_model_path = os.path.join(submission_path, model) - measurement_model_path = os.path.join(measurement_path, model) - compliance_model_path = os.path.join(compliance_path, model) code_model_path = os.path.join(code_path, model) scenarios = [ f for f in os.listdir(result_model_path) if not os.path.isfile( @@ -367,10 +360,6 @@ def generate_submission(env, state, inp, submission_division, logger): result_model_path, scenario) submission_scenario_path = os.path.join( submission_model_path, scenario) - measurement_scenario_path = os.path.join( - measurement_model_path, scenario) - compliance_scenario_path = os.path.join( - compliance_model_path, scenario) ''' if duplicate and scenario == 'singlestream': @@ -445,20 +434,11 @@ def generate_submission(env, state, inp, submission_division, logger): ) continue - if not os.path.isdir(measurement_scenario_path): - os.makedirs(measurement_scenario_path) - for mode in modes: result_mode_path = os.path.join(result_scenario_path, mode) submission_mode_path = os.path.join( submission_scenario_path, mode) - submission_measurement_path = measurement_scenario_path - submission_compliance_path = os.path.join( - compliance_scenario_path, mode) - if mode.startswith("TEST"): - submission_results_path = submission_compliance_path - else: - submission_results_path = submission_mode_path + submission_results_path = submission_mode_path if os.path.exists(submission_results_path): shutil.rmtree(submission_results_path) @@ -492,10 +472,10 @@ def generate_submission(env, state, inp, submission_division, logger): shutil.copy( analyzer_settings_file, os.path.join( - submission_measurement_path, "analyzer_table.md")) + submission_scenario_path, "analyzer_table.md")) shutil.copy( power_settings_file, os.path.join( - submission_measurement_path, "power_settings.md")) + submission_scenario_path, "power_settings.md")) result_ranging_path = os.path.join( result_mode_path, 'ranging') @@ -563,14 +543,14 @@ def generate_submission(env, state, inp, submission_division, logger): if os.path.exists(user_conf_path): shutil.copy( user_conf_path, os.path.join( - measurement_scenario_path, 'user.conf')) + submission_scenario_path, 'user.conf')) else: user_conf_path = os.path.join( result_mode_path, "user.conf") if os.path.exists(user_conf_path): shutil.copy( user_conf_path, os.path.join( - submission_measurement_path, 'user.conf')) + submission_scenario_path, 'user.conf')) else: if mode.lower() == "performance": return { @@ -579,7 +559,7 @@ def generate_submission(env, state, inp, submission_division, logger): # First check for measurements directory in scenario folder measurements_json_path = os.path.join( result_scenario_path, "measurements.json") - target_measurement_json_path = measurement_scenario_path + target_measurement_json_path = submission_scenario_path if not os.path.exists(measurements_json_path): measurements_json_path = os.path.join( @@ -668,7 +648,7 @@ def generate_submission(env, state, inp, submission_division, logger): shutil.copy( os.path.join( result_mode_path, f), os.path.join( - submission_measurement_path, f)) + submission_scenario_path, f)) if f == "system_info.txt" and not platform_info_file: # the first found system_info.txt will be taken as platform info file for a specific model to be placed in # measurements-model folder when generating @@ -679,7 +659,7 @@ def generate_submission(env, state, inp, submission_division, logger): shutil.copy( os.path.join( result_mode_path, f), os.path.join( - submission_measurement_path, mode + "_" + f)) + submission_scenario_path, mode + "_" + f)) if mode == "accuracy": if os.path.exists(os.path.join( @@ -705,13 +685,12 @@ def generate_submission(env, state, inp, submission_division, logger): result_scenario_path, "system_info.txt")): shutil.copy( os.path.join( - result_scenario_path, "system_info.txt"), os.path.join( - submission_measurement_path, f)) + result_scenario_path, "system_info.txt"), submission_scenario_path) platform_info_file = os.path.join( result_scenario_path, "system_info.txt") readme_file = os.path.join( - submission_measurement_path, "README.md") + submission_scenario_path, "README.md") if not os.path.exists(readme_file): with open(readme_file, mode='w') as f: f.write("TBD") # create an empty README @@ -741,7 +720,7 @@ def generate_submission(env, state, inp, submission_division, logger): shutil.copy( sys_info_file, os.path.join( - measurement_model_path, + submission_model_path, "system_info.txt")) # Copy system_info.txt to the submission measurements folder if any @@ -757,14 +736,14 @@ def generate_submission(env, state, inp, submission_division, logger): shutil.copy( sys_info_file, os.path.join( - measurement_path, + submission_path, "system_info.txt")) else: if env.get('MLC_GET_PLATFORM_DETAILS', '') == "yes": mlc_input = {'action': 'run', 'automation': 'script', 'tags': 'get,platform,details', - 'env': {'MLC_PLATFORM_DETAILS_FILE_PATH': os.path.join(measurement_path, "system_info.txt")}, + 'env': {'MLC_PLATFORM_DETAILS_FILE_PATH': os.path.join(submission_path, "system_info.txt")}, 'quiet': True } mlc = i['automation'].action_object @@ -779,7 +758,7 @@ def generate_submission(env, state, inp, submission_division, logger): logger.info(tabulate(result_table, headers=headers, tablefmt="pretty")) - sut_readme_file = os.path.join(measurement_path, "README.md") + sut_readme_file = os.path.join(submission_path, "README.md") with open(sut_readme_file, mode='w') as f: f.write(tabulate(result_table, headers=headers, tablefmt="github")) diff --git a/script/get-dataset-cnndm/meta.yaml b/script/get-dataset-cnndm/meta.yaml index 8214fe123..05de6ac0e 100644 --- a/script/get-dataset-cnndm/meta.yaml +++ b/script/get-dataset-cnndm/meta.yaml @@ -35,6 +35,10 @@ deps: skip_if_env: MLC_TMP_ML_MODEL: - llama3_1-8b +- tags: get,generic-python-lib,_package.transformers + skip_if_env: + MLC_TMP_ML_MODEL: + - llama3_1-8b - tags: get,generic-python-lib,_numpy skip_if_env: MLC_TMP_ML_MODEL: diff --git a/script/get-dataset-mlperf-inference-llama3/meta.yaml b/script/get-dataset-mlperf-inference-llama3/meta.yaml index 24aef9e20..929f03732 100644 --- a/script/get-dataset-mlperf-inference-llama3/meta.yaml +++ b/script/get-dataset-mlperf-inference-llama3/meta.yaml @@ -33,7 +33,6 @@ variations: default: true group: dataset-type env: - MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl MLC_DATASET_TYPE: validation MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl validation,rclone: @@ -42,12 +41,11 @@ variations: MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl validation,r2-downloader: env: - MLC_RCLONE_URL: "" - MLC_DATASET_FILE_NAME: "" + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/llama3-1-405b-dataset-8313.uri + MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl calibration: group: dataset-type env: - MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl MLC_DATASET_TYPE: calibration MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl calibration,rclone: @@ -56,8 +54,8 @@ variations: MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl calibration,r2-downloader: env: - MLC_RCLONE_URL: "" - MLC_DATASET_FILE_NAME: "" + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/llama3-1-405b-calibration-dataset-512.uri + MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl rclone: add_deps_recursive: dae: @@ -74,7 +72,6 @@ variations: tags: _r2-downloader env: MLC_DOWNLOAD_FILENAME: checkpoint - MLC_RCLONE_CONFIG_NAME: mlc-inference group: download-tool print_env_at_the_end: MLC_DATASET_LLAMA3_PATH: Path to the dataset diff --git a/script/get-dataset-mlperf-inference-mixtral/customize.py b/script/get-dataset-mlperf-inference-mixtral/customize.py index 5ecfbafdf..e233ebf50 100644 --- a/script/get-dataset-mlperf-inference-mixtral/customize.py +++ b/script/get-dataset-mlperf-inference-mixtral/customize.py @@ -19,9 +19,18 @@ def preprocess(i): def postprocess(i): env = i['env'] - env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = env['MLC_DATASET_PREPROCESSED_PATH'] - - if is_true(env.get('MLC_DATASET_MIXTRAL_GENERATE_TEST_DATA', '')): - env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = env['MLC_DATASET_MIXTRAL_TEST_DATA_GENERATED_PATH'] + if not env.get('MLC_DOWNLOAD_MODE', '') == "dry": + env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = env['MLC_DATASET_PREPROCESSED_PATH'] + + if env.get('MLC_DOWNLOAD_TOOL', '') == "r2-downloader": + if env.get('MLC_DATASET_TYPE', '') == 'validation': + env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = os.path.join( + env['MLC_DATASET_PREPROCESSED_PATH'], '09292024_mixtral_15k_mintoken2_v1.pkl') + elif env.get('MLC_DATASET_TYPE', '') == 'calibration': + env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = os.path.join( + env['MLC_DATASET_PREPROCESSED_PATH'], '2024.06.06_mixtral_15k_calibration_v4.pkl') + + if is_true(env.get('MLC_DATASET_MIXTRAL_GENERATE_TEST_DATA', '')): + env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] = env['MLC_DATASET_MIXTRAL_TEST_DATA_GENERATED_PATH'] return {'return': 0} diff --git a/script/get-dataset-mlperf-inference-mixtral/meta.yaml b/script/get-dataset-mlperf-inference-mixtral/meta.yaml index 744b5690d..a79b70dac 100644 --- a/script/get-dataset-mlperf-inference-mixtral/meta.yaml +++ b/script/get-dataset-mlperf-inference-mixtral/meta.yaml @@ -17,7 +17,7 @@ prehook_deps: _url.: - MLC_PACKAGE_URL names: - - download-file + - download-file-mixtral-dataset tags: - get - dataset-mixtral @@ -26,17 +26,24 @@ uid: 89e7c91444804775 variations: rclone: group: download-tool - default: true adr: - download-file: + download-file-mixtral-dataset: tags: _rclone env: MLC_DOWNLOAD_TOOL: rclone MLC_RCLONE_COPY_USING: copyurl + r2-downloader: + group: download-tool + default: true + adr: + download-file-mixtral-dataset: + tags: _r2-downloader + env: + MLC_DOWNLOAD_TOOL: r2-downloader wget: group: download-tool adr: - download-file: + download-file-mixtral-dataset: tags: _wget env: MLC_DOWNLOAD_TOOL: wget @@ -44,19 +51,31 @@ variations: default: true group: dataset-type adr: - download-file: + download-file-mixtral-dataset: extra_cache_tags: mixtral,get-mixtral-dataset,validation env: ML_MODEL_STARTING_WEIGHTS_FILENAME: https://github.com/mlcommons/inference/tree/master/language/mixtral-8x7b#preprocessed + MLC_DATASET_TYPE: validation + validation,rclone: + env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/mixtral_8x7b/09292024_mixtral_15k_mintoken2_v1.pkl + validation,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/mixtral-8x7b-validation-dataset.uri calibration: group: dataset-type adr: - download-file: + download-file-mixtral-dataset: extra_cache_tags: mixtral,get-mixtral-dataset,calibration env: ML_MODEL_STARTING_WEIGHTS_FILENAME: https://github.com/mlcommons/inference/tree/master/language/mixtral-8x7b#calibration-dataset + MLC_DATASET_TYPE: calibration + calibration,rclone: + env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/mixtral_8x7b%2F2024.06.06_mixtral_15k_calibration_v4.pkl + calibration,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/mixtral-8x7b-calibration-dataset.uri mlcommons-storage: env: MLC_DOWNLOAD_CHECKSUM: 78823c13e0e73e518872105c4b09628b @@ -72,3 +91,20 @@ variations: env: MLC_DATASET_MIXTRAL_GENERATE_TEST_DATA: 'yes' MLC_DATASET_MIXTRAL_TEST_DATA_SIZE: '#' + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run +tests: + run_inputs: + - variations_list: + - r2-downloader,calibration,dry-run + - rclone,calibration,dry-run + - r2-downloader,validation,dry-run + - rclone,validation,dry-run diff --git a/script/get-dataset-waymo-calibration/README.md b/script/get-dataset-waymo-calibration/README.md new file mode 100644 index 000000000..19a18b39e --- /dev/null +++ b/script/get-dataset-waymo-calibration/README.md @@ -0,0 +1,79 @@ +# README for get-dataset-waymo-calibration +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,waymo,dataset,calibration +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--waymo_calibration_path` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Dataset-format + +- `kitti_format` (default) + +### Download-src + +- `mlc` (default) + +### Download-tool + +- `r2-downloader` (default) +- `rclone` + +### Run-mode + +- `dry-run` diff --git a/script/get-dataset-waymo-calibration/meta.yaml b/script/get-dataset-waymo-calibration/meta.yaml index 96b916764..f952c7e1b 100644 --- a/script/get-dataset-waymo-calibration/meta.yaml +++ b/script/get-dataset-waymo-calibration/meta.yaml @@ -24,26 +24,16 @@ variations: MLC_DOWNLOAD_SRC: mlcommons group: download-src prehook_deps: - - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - true - tags: get,rclone - - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - true - force_cache: true - tags: get,rclone-config,_waymo - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_WAYMO_CALIBRATION_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_WAYMO_CALIBRATION_PATH - MLC_DOWNLOAD_URL: mlc_waymo:waymo_preprocessed_dataset/kitti_format/testing extra_cache_tags: waymo,dataset force_cache: true names: - - dae + - dae-waymo-calib tags: download-and-extract force_env_keys: - MLC_OUTDIRNAME @@ -53,9 +43,31 @@ variations: rclone: group: download-tool add_deps_recursive: - dae: + dae-waymo-calib: tags: _rclone + prehook_deps: + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - true + tags: get,rclone + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - true + force_cache: true + tags: get,rclone-config,_waymo + rclone,mlc: + env: + MLC_DOWNLOAD_URL: mlc_waymo:waymo_preprocessed_dataset/kitti_format/testing + r2-downloader: + group: download-tool + add_deps_recursive: + dae-waymo-calib: + tags: _r2-downloader default: true + r2-downloader,mlc: + env: + MLC_DOWNLOAD_URL: https://waymo.mlcommons-storage.org/metadata/calibration-dataset.uri + MLC_DOWNLOAD_FILENAME: calibration-dataset dry-run: group: run-mode env: @@ -63,4 +75,13 @@ variations: MLC_BYPASS_RCLONE_AUTH: True dry-run,rclone: env: - MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run \ No newline at end of file + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run +tests: + needs_pat: true + run_inputs: + - variations_list: + - r2-downloader,mlc,dry-run + - rclone,mlc,dry-run diff --git a/script/get-dataset-waymo/README.md b/script/get-dataset-waymo/README.md new file mode 100644 index 000000000..a9b89bde2 --- /dev/null +++ b/script/get-dataset-waymo/README.md @@ -0,0 +1,79 @@ +# README for get-dataset-waymo +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,dataset,waymo +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--waymo_path` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Dataset-format + +- `kitti_format` (default) + +### Download-src + +- `mlc` (default) + +### Download-tool + +- `r2-downloader` (default) +- `rclone` + +### Run-mode + +- `dry-run` diff --git a/script/get-dataset-waymo/meta.yaml b/script/get-dataset-waymo/meta.yaml index e90315017..953468644 100644 --- a/script/get-dataset-waymo/meta.yaml +++ b/script/get-dataset-waymo/meta.yaml @@ -76,3 +76,10 @@ variations: r2-downloader,mlc: env: MLC_DOWNLOAD_URL: https://waymo.mlcommons-storage.org/metadata/dataset.uri + MLC_DOWNLOAD_FILENAME: dataset +tests: + needs_pat: true + run_inputs: + - variations_list: + - r2-downloader,mlc,dry-run + - rclone,mlc,dry-run diff --git a/script/get-dataset-waymo/run.sh b/script/get-dataset-waymo/run.sh index 0af8d4a79..920224e60 100644 --- a/script/get-dataset-waymo/run.sh +++ b/script/get-dataset-waymo/run.sh @@ -9,6 +9,6 @@ if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" ]]; then cd "${MLC_DATASET_WAYMO_PATH}/training" || exit - for f in *.tar.gz; do tar -xzvf "$f"; done + for f in *.tar.gz; do tar --no-same-owner -xzvf "$f"; done cd - || exit -fi \ No newline at end of file +fi diff --git a/script/get-git-repo/meta.yaml b/script/get-git-repo/meta.yaml index e52110035..2bd9e61da 100644 --- a/script/get-git-repo/meta.yaml +++ b/script/get-git-repo/meta.yaml @@ -2,13 +2,14 @@ alias: get-git-repo automation_alias: script automation_uid: 5b4e0237da074764 cache: true +cache_expiration: 5d category: DevOps automation default_env: MLC_GIT_CHECKOUT_FOLDER: repo MLC_GIT_DEPTH: --depth 4 MLC_GIT_PATCH: 'no' MLC_GIT_RECURSE_SUBMODULES: ' --recurse-submodules' - MLC_GIT_URL: https://github.com/mlcommons/ck.git + MLC_GIT_URL: https://github.com/mlcommons/mlperf-automations deps: - tags: detect,os input_mapping: @@ -80,6 +81,7 @@ variations: MLC_GIT_URL: '#' group: repo sha.#: + cache_expiration: 500d default_variations: git-history: full-history env: @@ -94,6 +96,7 @@ variations: env: MLC_GIT_SUBMODULES: '#' tag.#: + cache_expiration: 500d env: MLC_GIT_CHECKOUT_TAG: '#' group: checkout diff --git a/script/get-ml-model-abtf-ssd-pytorch/README.md b/script/get-ml-model-abtf-ssd-pytorch/README.md index 6a6e5d631..7fc490756 100644 --- a/script/get-ml-model-abtf-ssd-pytorch/README.md +++ b/script/get-ml-model-abtf-ssd-pytorch/README.md @@ -69,6 +69,7 @@ mlcr get,ml-model,abtf-ssd-pytorch,ssd,resnet50,cmc ### Download-tool - `gdown` +- `r2-downloader` - `rclone` - `wget` (default) diff --git a/script/get-ml-model-abtf-ssd-pytorch/meta.yaml b/script/get-ml-model-abtf-ssd-pytorch/meta.yaml index fa1b5d39a..42f39b2f3 100644 --- a/script/get-ml-model-abtf-ssd-pytorch/meta.yaml +++ b/script/get-ml-model-abtf-ssd-pytorch/meta.yaml @@ -160,6 +160,20 @@ variations: rclone: group: download-tool + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config + force_cache: true + names: + - rclone-config + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + MLC_DOWNLOAD_SRC: + - mlcommons env: MLC_RCLONE_COPY_USING: copyurl adr: @@ -186,29 +200,55 @@ variations: env: MLC_MODEL_FORMAT: onnx - onnx,mlc: + onnx,mlc,rclone: env: MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_ssd/ssd_resnet50.onnx MLC_ML_MODEL_FILENAME: ssd_resnet50.onnx + MLC_DOWNLOAD_URL: 'mlc-cognata:mlc_cognata_dataset/<<>>' + MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' + + onnx,mlc,r2-downloader: + env: + MLC_ML_MODEL_FILENAME: ssd_resnet50.onnx + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/ssd_checkpoint_onnx.uri pytorch: group: model-format env: MLC_MODEL_FORMAT: pth - pytorch,mlc: + pytorch,mlc,rclone: env: MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_ssd/baseline_8MP_ss_scales_fm1_5x5_all_ep60.pth MLC_ML_MODEL_FILENAME: baseline_8MP_ss_scales_fm1_5x5_all_ep60.pth + MLC_DOWNLOAD_URL: 'mlc-cognata:mlc_cognata_dataset/<<>>' + MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' + + pytorch,mlc,r2-downloader: + env: + MLC_ML_MODEL_FILENAME: baseline_8MP_ss_scales_fm1_5x5_all_ep60.pth + MLC_DOWNLOAD_URL: 'https://cognata.mlcommons-storage.org/metadata/ssd_checkpoint_pth.uri' rclone,mlc: group: download-tool add_deps_recursive: - dae: + dae-ssd: tags: _rclone + rclone-config: + tags: _config-name.mlc-cognata + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 env: MLC_RCLONE_COPY_USING: sync + r2-downloader: + group: download-tool + + r2-downloader,mlc: + adr: + dae-ssd: + tags: _r2-downloader + dry-run: group: run-mode env: @@ -218,32 +258,23 @@ variations: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + mlc: group: download-src prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_config-name.mlc-cognata - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - env: - MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_ML_MODEL_SSD_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_MLC_MODEL_SSD_PATH - MLC_DOWNLOAD_URL: 'mlc-cognata:mlc_cognata_dataset/<<>>' - MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' extra_cache_tags: ml,model,ssd,resnet50 force_cache: true names: - - dae + - dae-ssd tags: download-and-extract force_env_keys: - MLC_OUTDIRNAME @@ -258,3 +289,5 @@ tests: - variations_list: - onnx,rclone,mlc,dry-run - pytorch,rclone,mlc,dry-run + - onnx,r2-downloader,mlc,dry-run + - pytorch,r2-downloader,mlc,dry-run diff --git a/script/get-ml-model-deeplabv3_plus/README.md b/script/get-ml-model-deeplabv3_plus/README.md index 438f35648..63a6cce07 100644 --- a/script/get-ml-model-deeplabv3_plus/README.md +++ b/script/get-ml-model-deeplabv3_plus/README.md @@ -63,6 +63,7 @@ No script specific inputs ### Download-tool +- `r2-downloader` - `rclone` (default) ### Model-format diff --git a/script/get-ml-model-deeplabv3_plus/meta.yaml b/script/get-ml-model-deeplabv3_plus/meta.yaml index 65c87e134..21184ac0b 100644 --- a/script/get-ml-model-deeplabv3_plus/meta.yaml +++ b/script/get-ml-model-deeplabv3_plus/meta.yaml @@ -20,48 +20,59 @@ variations: default: true env: MLC_MODEL_FORMAT: onnx - MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_deeplab/deeplabv3+_8mp.onnx MLC_ML_MODEL_FILENAME: deeplabv3+_8mp.onnx + onnx,mlc,rclone: + env: + MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_deeplab/deeplabv3+_8mp.onnx + onnx,mlc,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/deeplab_onnx.uri dynamic: {} onnx,dynamic: group: model-format env: MLC_MODEL_FORMAT: onnx - MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_deeplab/deeplabv3+_dynamic.onnx MLC_ML_MODEL_FILENAME: deeplabv3+_dynamic.onnx + onnx,dynamic,mlc,rclone: + env: + MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_deeplab/deeplabv3+_dynamic.onnx + onnx,dynamic,mlc,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/deeplab_dynamic_onnx.uri pytorch: group: model-format env: MLC_MODEL_FORMAT: pth - MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_deeplab/latest_deeplabv3plus_resnet50_cognata_os16_it100000.pth MLC_ML_MODEL_FILENAME: latest_deeplabv3plus_resnet50_cognata_os16_it100000.pth + pytorch,mlc,rclone: + env: + MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_deeplab/latest_deeplabv3plus_resnet50_cognata_os16_it100000.pth + pytorch,mlc,r2-downlaoder: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/deeplab_checkpoint.uri + rclone,mlc: + add_deps_recursive: + rclone-config: + tags: _config-name.mlc-cognata + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' + MLC_DOWNLOAD_URL: 'mlc-cognata:mlc_cognata_dataset/<<>>' mlc: group: download-src default: true prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_config-name.mlc-cognata - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - env: - MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_ML_MODEL_DEEPLABV3_PLUS_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_ML_MODEL_DEEPLABV3_PLUS_PATH - MLC_DOWNLOAD_URL: 'mlc-cognata:mlc_cognata_dataset/<<>>' - MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' extra_cache_tags: ml,model,deeplabv3,plus force_cache: true names: - - dae + - dae-deplabv3+ tags: download-and-extract force_env_keys: - MLC_OUTDIRNAME @@ -72,10 +83,27 @@ variations: MLC_DOWNLOAD_SRC: mlcommons rclone: group: download-tool + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config + force_cache: true + names: + - rclone-config + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes add_deps_recursive: - dae: + dae-deplabv3+: tags: _rclone default: true + r2-downloader: + group: download-tool + add_deps_recursive: + dae-deplabv3+: + tags: _r2-downloader dry-run: group: run-mode env: @@ -83,6 +111,9 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x tests: needs_pat: true run_inputs: @@ -90,3 +121,6 @@ tests: - onnx,rclone,mlc,dry-run - onnx_dynamic,rclone,mlc,dry-run - pytorch,rclone,mlc,dry-run + - onnx,r2-downloader,mlc,dry-run + - onnx_dynamic,r2-downloader,mlc,dry-run + - pytorch,r2-downloader,mlc,dry-run diff --git a/script/get-ml-model-deepseek-r1/COPYRIGHT.md b/script/get-ml-model-deepseek-r1/COPYRIGHT.md new file mode 100644 index 000000000..d2ceead84 --- /dev/null +++ b/script/get-ml-model-deepseek-r1/COPYRIGHT.md @@ -0,0 +1,9 @@ +# Copyright Notice + +© 2025-2026 MLCommons. All Rights Reserved. + +This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: + +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. diff --git a/script/get-ml-model-deepseek-r1/README.md b/script/get-ml-model-deepseek-r1/README.md new file mode 100644 index 000000000..e3edc0e61 --- /dev/null +++ b/script/get-ml-model-deepseek-r1/README.md @@ -0,0 +1,70 @@ +# README for get-ml-model-deepseek-r1 +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,ml-model,deepseek-r1,get-ml-model-deepseek-r1 +``` + +No script specific inputs +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Download-src + +- `mlc` (default) + +### Download-tool + +- `r2-downloader` (default) + +### Run-mode + +- `dry-run` diff --git a/script/get-ml-model-deepseek-r1/customize.py b/script/get-ml-model-deepseek-r1/customize.py new file mode 100644 index 000000000..5f0d910fe --- /dev/null +++ b/script/get-ml-model-deepseek-r1/customize.py @@ -0,0 +1,31 @@ +from mlc import utils +import os +from utils import is_true + + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + if os_info['platform'] == "windows": + return {'return': 1, 'error': 'Script not supported in windows yet!'} + + if env.get('MLC_ML_MODEL_DEEPSEEK_R1_PATH', '') == '': + env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes" + elif not os.path.exists(env['MLC_ML_MODEL_DEEPSEEK_R1_PATH']): + return { + 'return': 1, 'error': f"Path {env['MLC_ML_MODEL_DEEPSEEK_R1_PATH']} does not exists!"} + + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + + if not env.get('MLC_DOWNLOAD_MODE', '') == "dry": + env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['MLC_ML_MODEL_DEEPSEEK_R1_PATH'] + + return {'return': 0} diff --git a/script/get-ml-model-deepseek-r1/meta.yaml b/script/get-ml-model-deepseek-r1/meta.yaml new file mode 100644 index 000000000..c055364ef --- /dev/null +++ b/script/get-ml-model-deepseek-r1/meta.yaml @@ -0,0 +1,58 @@ +alias: get-ml-model-deepseek-r1 +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +new_env_keys: +- MLC_ML_MODEL_DEEPSEEK_R1_PATH +- MLC_ML_MODEL_FILE_WITH_PATH +print_env_at_the_end: + MLC_ML_MODEL_DEEPSEEK_R1_PATH: DeepSeek-R1 checkpoint path +tags: +- get +- ml-model +- deepseek-r1 +- get-ml-model-deepseek-r1 +tests: + run_inputs: + - variations_list: + - r2-downloader,mlc,dry-run +uid: 2bcb61611b1f4af3 +variations: + dry-run: + env: + MLC_DOWNLOAD_MODE: dry + group: run-mode + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + mlc: + default: true + env: + MLC_DOWNLOAD_SRC: mlcommons + group: download-src + prehook_deps: + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_ML_MODEL_DEEPSEEK_R1_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_ML_MODEL_DEEPSEEK_R1_PATH + extra_cache_tags: ml,model,deepseek-r1 + force_cache: true + force_env_keys: + - MLC_OUTDIRNAME + names: + - dae-deepseek-r1 + tags: download-and-extract + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL + r2-downloader: + default: true + add_deps_recursive: + dae-deepseek-r1: + tags: _r2-downloader + group: download-tool + r2-downloader,mlc: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/deepseek-r1-0528.uri diff --git a/script/get-ml-model-dlrm-terabyte/README.md b/script/get-ml-model-dlrm-terabyte/README.md new file mode 100644 index 000000000..e4eb03be6 --- /dev/null +++ b/script/get-ml-model-dlrm-terabyte/README.md @@ -0,0 +1,91 @@ +# README for get-ml-model-dlrm-terabyte +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,ml-model,dlrm,raw,terabyte,criteo-terabyte,criteo,recommendation +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--dir` | | | `` | +| `--download_path` | Alias for dir | | `` | +| `--to` | Alias for dir | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Download-tool + +- `r2-downloader` +- `rclone` +- `wget` + +### Framework + +- `onnx` +- `pytorch` (default) + +### Precision + +- `fp32` (default) + +### Run-mode + +- `dry-run` + +### Type + +- `weight_sharded` (default) + +### Ungrouped + +- `debug` diff --git a/script/get-ml-model-dlrm-terabyte/meta.yaml b/script/get-ml-model-dlrm-terabyte/meta.yaml index fed99aaa4..86b6bfa90 100644 --- a/script/get-ml-model-dlrm-terabyte/meta.yaml +++ b/script/get-ml-model-dlrm-terabyte/meta.yaml @@ -93,7 +93,7 @@ variations: MLC_PACKAGE_URL: https://dlrm.s3-us-west-1.amazonaws.com/models/tb0875_10M.pt pytorch,fp32,weight_sharded: default_variations: - download-tool: rclone + download-tool: r2-downloader env: MLC_DOWNLOAD_CHECKSUM: '' MLC_ML_MODEL_ACCURACY: '0.8025' @@ -104,6 +104,9 @@ variations: env: MLC_PACKAGE_URL: mlc-inference:mlcommons-inference-wg-public/model_weights MLC_RCLONE_CONFIG_NAME: mlc-inference + pytorch,fp32,weight_sharded,r2-downloader: + env: + MLC_PACKAGE_URL: https://inference.mlcommons-storage.org/metadata/dlrm-v2-model-weights.uri pytorch,fp32,weight_sharded,wget: env: MLC_DAE_EXTRACT_DOWNLOADED: 'yes' @@ -115,6 +118,21 @@ variations: dae: tags: _rclone group: download-tool + r2-downloader: + ad: + dae: + tags: _r2-downloader + group: download-tool + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x weight_sharded: default: true env: @@ -125,3 +143,8 @@ variations: dae: tags: _wget group: download-tool +tests: + run_inputs: + - variations_list: + - pytorch,fp32,weight_sharded,r2-downloader,dry-run + - pytorch,fp32,weight_sharded,rclone,dry-run \ No newline at end of file diff --git a/script/get-ml-model-gptj/README.md b/script/get-ml-model-gptj/README.md new file mode 100644 index 000000000..76eaa9c8b --- /dev/null +++ b/script/get-ml-model-gptj/README.md @@ -0,0 +1,97 @@ +# README for get-ml-model-gptj +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,raw,ml-model,gptj,gpt-j,large-language-model +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--checkpoint` | | | `` | +| `--download_path` | | | `` | +| `--to` | Alias for download_path | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Download-tool + +- `r2-downloader` (default) +- `rclone` +- `wget` + +### Framework + +- `pytorch` (default) +- `saxml` + +### Model-provider + +- `intel` +- `mlcommons` (default) +- `nvidia` + +### Precision + +- `fp32` +- `fp8` +- `int4` +- `int8` +- `uint8` + +### Run-mode + +- `dry-run` + +### Ungrouped + +- `batch_size.#` _(# can be substituted dynamically)_ diff --git a/script/get-ml-model-gptj/customize.py b/script/get-ml-model-gptj/customize.py index 8d0ca4e9c..6a88b7fe1 100644 --- a/script/get-ml-model-gptj/customize.py +++ b/script/get-ml-model-gptj/customize.py @@ -67,39 +67,42 @@ def postprocess(i): env = i['env'] - if os.path.exists(os.path.join( - env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final")): - env['GPTJ_CHECKPOINT_PATH'] = os.path.join( - env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final") - - is_saxml = env.get('MLC_TMP_MODEL_SAXML', '') - if is_saxml == "fp32": - if os.path.exists("pax_gptj_checkpoint"): - env['GPTJ_SAXML_CHECKPOINT_PATH'] = os.path.join( - os.getcwd(), "pax_gptj_checkpoint") - env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_SAXML_CHECKPOINT_PATH'] - else: - return {'return': 1, 'error': 'pax_gptj_checkpoint generation failed'} + if not env.get('MLC_DOWNLOAD_MODE', '') == "dry": + if os.path.exists(os.path.join( + env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final")): + env['GPTJ_CHECKPOINT_PATH'] = os.path.join( + env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final") + + is_saxml = env.get('MLC_TMP_MODEL_SAXML', '') + if is_saxml == "fp32": + if os.path.exists("pax_gptj_checkpoint"): + env['GPTJ_SAXML_CHECKPOINT_PATH'] = os.path.join( + os.getcwd(), "pax_gptj_checkpoint") + env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_SAXML_CHECKPOINT_PATH'] + else: + return {'return': 1, + 'error': 'pax_gptj_checkpoint generation failed'} - elif is_saxml == "int8": - if os.path.exists("int8_ckpt"): - env['GPTJ_SAXML_INT8_CHECKPOINT_PATH'] = os.path.join( - os.getcwd(), "int8_ckpt") - env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_SAXML_INT8_CHECKPOINT_PATH'] + elif is_saxml == "int8": + if os.path.exists("int8_ckpt"): + env['GPTJ_SAXML_INT8_CHECKPOINT_PATH'] = os.path.join( + os.getcwd(), "int8_ckpt") + env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_SAXML_INT8_CHECKPOINT_PATH'] + else: + return {'return': 1, + 'error': 'pax_gptj_checkpoint generation failed'} + elif env.get('MLC_TMP_ML_MODEL_PROVIDER', '') == 'nvidia': + env['MLC_ML_MODEL_FILE_WITH_PATH'] = os.path.join( + env['MLC_NVIDIA_MLPERF_SCRATCH_PATH'], + 'models', + 'GPTJ-6B', + 'fp8-quantized-ammo', + 'GPTJ-FP8-quantized') else: - return {'return': 1, 'error': 'pax_gptj_checkpoint generation failed'} - elif env.get('MLC_TMP_ML_MODEL_PROVIDER', '') == 'nvidia': - env['MLC_ML_MODEL_FILE_WITH_PATH'] = os.path.join( - env['MLC_NVIDIA_MLPERF_SCRATCH_PATH'], - 'models', - 'GPTJ-6B', - 'fp8-quantized-ammo', - 'GPTJ-FP8-quantized') - else: - env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_CHECKPOINT_PATH'] + env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_CHECKPOINT_PATH'] - env['MLC_ML_MODEL_FILE'] = os.path.basename( - env['MLC_ML_MODEL_FILE_WITH_PATH']) - env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_ML_MODEL_FILE_WITH_PATH'] + env['MLC_ML_MODEL_FILE'] = os.path.basename( + env['MLC_ML_MODEL_FILE_WITH_PATH']) + env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_ML_MODEL_FILE_WITH_PATH'] return {'return': 0} diff --git a/script/get-ml-model-gptj/meta.yaml b/script/get-ml-model-gptj/meta.yaml index 3c7e86cb8..ea5c2dc97 100644 --- a/script/get-ml-model-gptj/meta.yaml +++ b/script/get-ml-model-gptj/meta.yaml @@ -37,7 +37,8 @@ prehook_deps: tests: run_inputs: - variations_list: - - fp32 + - fp32,pytorch,r2-downloader,dry-run + - fp32,pytorch,rclone,dry-run print_env_at_the_end: @@ -102,10 +103,15 @@ variations: env: MLC_DOWNLOAD_CHECKSUM_NOT_USED: e677e28aaf03da84584bb3073b7ee315 MLC_PACKAGE_URL: https://cloud.mlcommons.org/index.php/s/QAZ2oM94MkFtbQx/download - MLC_RCLONE_CONFIG_NAME: mlc-inference - MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/gpt-j MLC_UNZIP: 'yes' required_disk_space: 22700 + pytorch,fp32,rclone: + env: + MLC_PACKAGE_URL: mlc-inference:mlcommons-inference-wg-public/gpt-j + MLC_RCLONE_CONFIG_NAME: mlc-inference + pytorch,fp32,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/gpt-j-model-checkpoint.uri pytorch,fp32,wget: add_deps_recursive: dae: @@ -127,7 +133,7 @@ variations: MLC_GPTJ_INTEL_MODEL: '' force_new_env_keys: - GPTJ_CHECKPOINT_PATH - tags: get,ml-model,gpt-j,_fp32,_pytorch + tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader - tags: get,conda,_name.gptj-pt - tags: get,python,_conda.gptj-pt - names: @@ -166,7 +172,7 @@ variations: - env: {} force_new_env_keys: - GPTJ_CHECKPOINT_PATH - tags: get,ml-model,gpt-j,_fp32,_pytorch + tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader - names: - nvidia-inference-common-code tags: get,nvidia,inference,common-code @@ -183,16 +189,33 @@ variations: add_deps_recursive: dae: tags: _rclone - default: true env: MLC_DOWNLOAD_FILENAME: checkpoint - MLC_DOWNLOAD_URL: <<>> + MLC_DOWNLOAD_URL: <<>> group: download-tool + r2-downloader: + group: download-tool + default: true + add_deps_recursive: + dae: + tags: _r2-downloader + env: + MLC_DOWNLOAD_FILENAME: checkpoint + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run saxml: group: framework saxml,fp32: deps: - - tags: get,ml-model,gptj,_pytorch,_fp32 + - tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader - names: - python - python3 diff --git a/script/get-ml-model-llama3/meta.yaml b/script/get-ml-model-llama3/meta.yaml index b10665d20..d316d1f45 100644 --- a/script/get-ml-model-llama3/meta.yaml +++ b/script/get-ml-model-llama3/meta.yaml @@ -74,8 +74,8 @@ variations: enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - yes - default: true r2-downloader: + default: true group: download-tool add_deps_recursive: dae: diff --git a/script/get-ml-model-mixtral/README.md b/script/get-ml-model-mixtral/README.md new file mode 100644 index 000000000..1649edfc4 --- /dev/null +++ b/script/get-ml-model-mixtral/README.md @@ -0,0 +1,88 @@ +# README for get-ml-model-mixtral +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,raw,ml-model,language-processing,mixtral,mixtral-8x7b +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--checkpoint` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Download-tool + +- `r2-downloader` + +### Framework + +- `pytorch` (default) + +### Huggingface-stub + +- `mistralai/Mixtral-8x7B-Instruct-v0.1` (default) +- `stub.#` _(# can be substituted dynamically)_ + +### Precision + +- `fp32` (default) + +### Run-mode + +- `dry-run` + +### Ungrouped + +- `batch_size.#` _(# can be substituted dynamically)_ +- `mlc` diff --git a/script/get-ml-model-mixtral/customize.py b/script/get-ml-model-mixtral/customize.py index d33656b33..20adf59f4 100644 --- a/script/get-ml-model-mixtral/customize.py +++ b/script/get-ml-model-mixtral/customize.py @@ -19,10 +19,11 @@ def postprocess(i): env = i['env'] - if env.get('MIXTRAL_CHECKPOINT_PATH', '') == '': - env['MIXTRAL_CHECKPOINT_PATH'] = env['MLC_ML_MODEL_PATH'] - else: - env['MLC_ML_MODEL_PATH'] = env['MIXTRAL_CHECKPOINT_PATH'] - env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_ML_MODEL_PATH'] + if not env.get('MLC_DOWNLOAD_MODE', '') == "dry": + if env.get('MIXTRAL_CHECKPOINT_PATH', '') == '': + env['MIXTRAL_CHECKPOINT_PATH'] = env['MLC_ML_MODEL_PATH'] + else: + env['MLC_ML_MODEL_PATH'] = env['MIXTRAL_CHECKPOINT_PATH'] + env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_ML_MODEL_PATH'] return {'return': 0} diff --git a/script/get-ml-model-mixtral/meta.yaml b/script/get-ml-model-mixtral/meta.yaml index 48ded49db..804f33e4f 100644 --- a/script/get-ml-model-mixtral/meta.yaml +++ b/script/get-ml-model-mixtral/meta.yaml @@ -17,6 +17,9 @@ prehook_deps: - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' + skip_if_env: + MLC_DOWNLOAD_TOOL: + - r2-downloader env: {} extra_cache_tags: mixtral force_env_keys: @@ -57,6 +60,42 @@ variations: MLC_ML_MODEL_STARTING_WEIGHTS_FILENAME: https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1 MLC_MODEL_ZOO_ENV_KEY: MIXTRAL group: huggingface-stub + mlc: + prehook_deps: + - env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MIXTRAL_CHECKPOINT_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MIXTRAL_CHECKPOINT_PATH + extra_cache_tags: ml-model,mixtral,raw,ml-model-mixtral + force_cache: true + names: + - dae-mixtral-model + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_PACKAGE_URL + mlc,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/mixtral-8x7b-model-checkpoint.uri + MLC_ML_MODEL_STARTING_WEIGHTS_FILENAME: "" + r2-downloader: + adr: + dae-mixtral-model: + tags: _r2-downloader + env: + MLC_DOWNLOAD_TOOL: r2-downloader + group: download-tool + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x pytorch: default: true env: @@ -69,3 +108,7 @@ variations: env: MLC_MODEL_ZOO_ENV_KEY: MIXTRAL group: huggingface-stub +tests: + run_inputs: + - variations_list: + - r2-downloader,mlc,dry-run diff --git a/script/get-ml-model-pointpainting/README.md b/script/get-ml-model-pointpainting/README.md new file mode 100644 index 000000000..7e59ec1e6 --- /dev/null +++ b/script/get-ml-model-pointpainting/README.md @@ -0,0 +1,81 @@ +# README for get-ml-model-pointpillars +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,ml-model,ml,model,pointpainting +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--pp_path` | | | `` | +| `--dp_resnet50_path` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Device + +- `cpu` +- `gpu` (default) + +### Download-src + +- `mlc` (default) + +### Download-tool + +- `r2-downloader` (default) +- `rclone` + +### Run-mode + +- `dry-run` diff --git a/script/get-ml-model-pointpainting/meta.yaml b/script/get-ml-model-pointpainting/meta.yaml index 8047b6188..7ed3a646f 100644 --- a/script/get-ml-model-pointpainting/meta.yaml +++ b/script/get-ml-model-pointpainting/meta.yaml @@ -86,3 +86,10 @@ variations: r2-downloader,mlc: env: MLC_DOWNLOAD_URL: https://waymo.mlcommons-storage.org/metadata/model.uri + MLC_DOWNLOAD_FILENAME: model +tests: + needs_pat: true + run_inputs: + - variations_list: + - r2-downloader,mlc,dry-run + - rclone,mlc,dry-run \ No newline at end of file diff --git a/script/get-ml-model-rgat/README.md b/script/get-ml-model-rgat/README.md new file mode 100644 index 000000000..957bb745f --- /dev/null +++ b/script/get-ml-model-rgat/README.md @@ -0,0 +1,83 @@ +# README for get-ml-model-rgat +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,raw,ml-model,rgat +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--checkpoint` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Download-source + +- `mlcommons` (default) + +### Download-tool + +- `r2-downloader` +- `rclone` + +### Precision + +- `fp32` (default) + +### Run-mode + +- `dry-run` + +### Ungrouped + +- `pytorch` diff --git a/script/get-ml-model-rgat/meta.yaml b/script/get-ml-model-rgat/meta.yaml index b9ffe25c1..2ab088b9f 100644 --- a/script/get-ml-model-rgat/meta.yaml +++ b/script/get-ml-model-rgat/meta.yaml @@ -15,8 +15,6 @@ new_env_keys: - RGAT_CHECKPOINT_PATH prehook_deps: - enable_if_env: - MLC_DOWNLOAD_TOOL: - - rclone MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' env: @@ -24,7 +22,7 @@ prehook_deps: extra_cache_tags: rgat,gnn,model,ml-model force_cache: true names: - - download-file + - download-rgat-model-file tags: download,file force_env_keys: - MLC_OUTDIRNAME @@ -52,18 +50,45 @@ variations: mlcommons: default: true default_variations: - download-tool: rclone + download-tool: r2-downloader group: download-source rclone: adr: - download-file: + download-rgat-model-file: tags: _rclone env: MLC_DOWNLOAD_TOOL: rclone MLC_RCLONE_CONFIG_NAME: mlc-inference group: download-tool - rclone,fp32: + r2-downloader: + adr: + download-rgat-model-file: + tags: _r2-downloader + env: + MLC_DOWNLOAD_TOOL: r2-downloader + group: download-tool + rclone,mlcommons,fp32: env: MLC_ML_MODEL_STARTING_WEIGHTS_FILENAME: https://github.com/mlcommons/inference/tree/master/graph/R-GAT#download-model-using-rclone MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/R-GAT/RGAT.pt MLC_DOWNLOAD_FILENAME: RGAT + r2-downloader,mlcommons,fp32: + env: + MLC_ML_MODEL_STARTING_WEIGHTS_FILENAME: https://github.com/mlcommons/inference/tree/master/graph/R-GAT#download-model-using-mlc-r2-downloader + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/rgat-model.uri + MLC_DOWNLOAD_FILENAME: RGAT + dry-run: + env: + MLC_DOWNLOAD_MODE: dry + group: run-mode + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x +tests: + run_inputs: + - variations_list: + - rclone,mlcommons,fp32,dry-run + - r2-downloader,mlcommons,fp32,dry-run diff --git a/script/get-ml-model-stable-diffusion/README.md b/script/get-ml-model-stable-diffusion/README.md new file mode 100644 index 000000000..85a3359bd --- /dev/null +++ b/script/get-ml-model-stable-diffusion/README.md @@ -0,0 +1,95 @@ +# README for get-ml-model-stable-diffusion +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,raw,ml-model,stable-diffusion,sdxl,text-to-image +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--checkpoint` | | | `` | +| `--download_path` | | | `` | +| `--to` | Alias for download_path | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Download-source + +- `huggingface` +- `mlcommons` (default) + +### Download-tool + +- `git` +- `r2-downloader` +- `rclone` +- `wget` + +### Framework + +- `pytorch` (default) + +### Precision + +- `fp16` +- `fp32` (default) +- `int8` +- `uint8` + +### Run-mode + +- `dry-run` + +### Ungrouped + +- `batch_size.#` _(# can be substituted dynamically)_ diff --git a/script/get-ml-model-stable-diffusion/customize.py b/script/get-ml-model-stable-diffusion/customize.py index 1eafee100..c46aab699 100644 --- a/script/get-ml-model-stable-diffusion/customize.py +++ b/script/get-ml-model-stable-diffusion/customize.py @@ -19,11 +19,12 @@ def postprocess(i): env = i['env'] - if env.get('SDXL_CHECKPOINT_PATH', '') == '': - env['SDXL_CHECKPOINT_PATH'] = env['MLC_ML_MODEL_PATH'] - elif env.get('MLC_ML_MODEL_PATH', '') == '': - env['MLC_ML_MODEL_PATH'] = env['SDXL_CHECKPOINT_PATH'] + if env.get('MLC_DOWNLOAD_MODE', '') != "dry": + if env.get('SDXL_CHECKPOINT_PATH', '') == '': + env['SDXL_CHECKPOINT_PATH'] = env['MLC_ML_MODEL_PATH'] + elif env.get('MLC_ML_MODEL_PATH', '') == '': + env['MLC_ML_MODEL_PATH'] = env['SDXL_CHECKPOINT_PATH'] - env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['SDXL_CHECKPOINT_PATH'] + env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['SDXL_CHECKPOINT_PATH'] return {'return': 0} diff --git a/script/get-ml-model-stable-diffusion/meta.yaml b/script/get-ml-model-stable-diffusion/meta.yaml index a1c96bd8b..52f8b4a7c 100644 --- a/script/get-ml-model-stable-diffusion/meta.yaml +++ b/script/get-ml-model-stable-diffusion/meta.yaml @@ -31,8 +31,8 @@ prehook_deps: force_env_keys: - MLC_OUTDIRNAME - enable_if_env: - MLC_DOWNLOAD_TOOL: - - rclone + MLC_DOWNLOAD_SOURCE: + - mlcommons MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' env: @@ -40,7 +40,7 @@ prehook_deps: extra_cache_tags: stable-diffusion,sdxl,model force_cache: true names: - - dae + - dae-sdxl-model tags: download-and-extract update_tags_from_env_with_prefix: _url.: @@ -89,7 +89,9 @@ variations: mlcommons: default: true default_variations: - download-tool: rclone + download-tool: r2-downloader + env: + MLC_DOWNLOAD_SOURCE: mlcommons group: download-source pytorch: default: true @@ -104,18 +106,33 @@ variations: required_disk_space: 13000 rclone: adr: - dae: + dae-sdxl-model: tags: _rclone env: MLC_DOWNLOAD_TOOL: rclone MLC_RCLONE_CONFIG_NAME: mlc-inference group: download-tool + r2-downloader: + adr: + dae-sdxl-model: + tags: _r2-downloader + env: + MLC_DOWNLOAD_TOOL: r2-downloader + group: download-tool rclone,fp16: env: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/stable_diffusion_fp16 rclone,fp32: env: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/stable_diffusion_fp32 + r2-downloader,fp16: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/stable-diffusion-xl-1-0-fp16-checkpoint.uri + MLC_DOWNLOAD_FILENAME: fp16-checkpoint + r2-downloader,fp32: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/stable-diffusion-xl-1-0-fp32-checkpoint.uri + MLC_DOWNLOAD_FILENAME: fp32-checkpoint uint8: env: MLC_ML_MODEL_INPUT_DATA_TYPES: uint8 @@ -124,8 +141,23 @@ variations: group: precision wget: adr: - dae: + dae-sdxl-model: tags: _wget env: MLC_DOWNLOAD_TOOL: wget group: download-tool + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x +test: + run_inputs: + - variations_list: + - fp32,r2-downloader,dry-run + - fp16,r2-downloader,dry-run diff --git a/script/get-mlperf-inference-utils/mlperf_utils.py b/script/get-mlperf-inference-utils/mlperf_utils.py index ffb0d3d7a..d35974301 100644 --- a/script/get-mlperf-inference-utils/mlperf_utils.py +++ b/script/get-mlperf-inference-utils/mlperf_utils.py @@ -188,7 +188,8 @@ def get_result_string(version, model, scenario, result_path, has_power, sub_res, result['performance'] = round(performance_result_, 3) if scenario != effective_scenario: - inferred, inferred_result = checker.get_inferred_result( + print(f"{fname} : {scenario} : {effective_scenario}") + inferred, inferred_result, inferred_valid = checker.get_inferred_result( scenario, effective_scenario, performance_result, mlperf_log, config, False) if has_power: diff --git a/script/get-preprocessed-dataset-cognata/README.md b/script/get-preprocessed-dataset-cognata/README.md new file mode 100644 index 000000000..84273df95 --- /dev/null +++ b/script/get-preprocessed-dataset-cognata/README.md @@ -0,0 +1,85 @@ +# README for get-preprocessed-dataset-cognata +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,dataset,cognata,preprocessed +``` + +No script specific inputs +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Dataset-src + +- `prebuilt` (default) + +### Dataset-type + +- `calibration` +- `validation` (default) + +### Download-src + +- `mlc` (default) + +### Download-tool + +- `r2-downloader` +- `rclone` (default) + +### Run-mode + +- `dry-run` + +### Task + +- `2d_obj_det` (default) +- `segmentation` diff --git a/script/get-preprocessed-dataset-cognata/meta.yaml b/script/get-preprocessed-dataset-cognata/meta.yaml index 940351310..c33caf57a 100644 --- a/script/get-preprocessed-dataset-cognata/meta.yaml +++ b/script/get-preprocessed-dataset-cognata/meta.yaml @@ -38,22 +38,42 @@ variations: env: MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAME: val_2d MLC_DATASET_COGNATA_TAR_FILENAME: val_2d.tar.gz + validation,2d_obj_det,rclone: + env: MLC_DOWNLOAD_URL: mlc-cognata:mlc_cognata_dataset/preprocessed_2d/<<>> + validation,2d_obj_det,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/val_2d.uri calibration,2d_obj_det: env: MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAME: calib_2d MLC_DATASET_COGNATA_TAR_FILENAME: calib_2d.tar.gz + calibration,2d_obj_det,rclone: + env: MLC_DOWNLOAD_URL: mlc-cognata:mlc_cognata_dataset/preprocessed_2d/<<>> + calibration,2d_obj_det,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/calib_2d.uri validation,segmentation: env: MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAME: val_seg MLC_DATASET_COGNATA_TAR_FILENAME: val_seg.tar.gz + validation,segmentation,rclone: + env: MLC_DOWNLOAD_URL: mlc-cognata:mlc_cognata_dataset/preprocessed_seg/<<>> + validation,segmentation,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/val_seg.uri calibration,segmentation: env: MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAME: calib_seg MLC_DATASET_COGNATA_TAR_FILENAME: calib_seg.tar.gz + calibration,segmentation,rclone: + env: MLC_DOWNLOAD_URL: mlc-cognata:mlc_cognata_dataset/preprocessed_seg/<<>> + calibration,segmentation,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/calib_seg.uri prebuilt: default: true group: dataset-src @@ -61,22 +81,13 @@ variations: MLC_NUSCENES_DATASET_TYPE: prebuilt MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_COGNATA_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_COGNATA_PATH + prebuilt,rclone: + env: MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' mlc: group: download-src default: true prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_config-name.mlc-cognata - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - env: - MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' @@ -92,6 +103,19 @@ variations: - MLC_DOWNLOAD_URL env: MLC_DOWNLOAD_SRC: mlcommons + mlc,rclone: + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_config-name.mlc-cognata + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 rclone: group: download-tool add_deps_recursive: @@ -105,10 +129,23 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + r2-downloader: + group: download-tool + add_deps_recursive: + dae: + tags: _r2-downloader + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x tests: + needs_pat: true run_inputs: - variations_list: - validation,prebuilt,2d_obj_det,rclone,mlc,dry-run - calibration,prebuilt,2d_obj_det,rclone,mlc,dry-run - validation,prebuilt,segmentation,rclone,mlc,dry-run - - calibration,prebuilt,segmentation,rclone,mlc,dry-run \ No newline at end of file + - calibration,prebuilt,segmentation,rclone,mlc,dry-run + - validation,prebuilt,2d_obj_det,r2-downloader,mlc,dry-run + - calibration,prebuilt,2d_obj_det,r2-downloader,mlc,dry-run + - validation,prebuilt,segmentation,r2-downloader,mlc,dry-run + - calibration,prebuilt,segmentation,r2-downloader,mlc,dry-run diff --git a/script/get-preprocessed-dataset-cognata/run.sh b/script/get-preprocessed-dataset-cognata/run.sh index 0c141b8e6..4e9dbe810 100644 --- a/script/get-preprocessed-dataset-cognata/run.sh +++ b/script/get-preprocessed-dataset-cognata/run.sh @@ -3,7 +3,7 @@ if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" ]]; then cd "${MLC_PREPROCESSED_DATASET_COGNATA_PATH}" || exit for f in *.tar.gz; do - tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + tar --no-same-owner -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } done cd - || exit fi \ No newline at end of file diff --git a/script/get-preprocessed-dataset-criteo/README.md b/script/get-preprocessed-dataset-criteo/README.md new file mode 100644 index 000000000..8141649ac --- /dev/null +++ b/script/get-preprocessed-dataset-criteo/README.md @@ -0,0 +1,90 @@ +# README for get-preprocessed-dataset-criteo +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,dataset,criteo,recommendation,dlrm,preprocessed +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--dir` | | | `` | +| `--output_dir` | | | `` | +| `--threads` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Download-tool + +- `r2-downloader` +- `rclone` (default) + +### Run-mode + +- `dry-run` + +### Src + +- `mlc` (default) +- `preprocess` + +### Type + +- `multihot` (default) + +### Ungrouped + +- `1` +- `50` +- `fake` +- `full` +- `validation` diff --git a/script/get-preprocessed-dataset-criteo/meta.yaml b/script/get-preprocessed-dataset-criteo/meta.yaml index 7455121c2..0fa577d02 100644 --- a/script/get-preprocessed-dataset-criteo/meta.yaml +++ b/script/get-preprocessed-dataset-criteo/meta.yaml @@ -137,25 +137,37 @@ variations: multihot,mlc: deps: - env: - MLC_DOWNLOAD_CHECKSUM_FILE: <<>>/checksums.txt MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_PREPROCESSED_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_PREPROCESSED_PATH MLC_EXTRACT_TO_FOLDER: criteo-preprocessed - MLC_RCLONE_CONFIG_NAME: mlc-inference - MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/dlrm_preprocessed extra_cache_tags: criteo,preprocessed,dataset force_cache: true + force_env_keys: + - MLC_OUTDIRNAME names: - - dae + - dae-criteo-preprocessed tags: download-and-extract update_tags_from_env_with_prefix: _url.: - MLC_DOWNLOAD_URL + multihot,mlc,rclone: + env: + MLC_DOWNLOAD_CHECKSUM_FILE: <<>>/checksums.txt + MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/dlrm_preprocessed + MLC_RCLONE_CONFIG_NAME: mlc-inference + multihot,mlc,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/dlrm-v2-preprocessed-dataset.uri rclone: group: download-tool add_deps_recursive: - dae: + dae-criteo-preprocessed: tags: _rclone + r2-downloader: + group: download-tool + add_deps_recursive: + dae-criteo-preprocessed: + tags: _r2-downloader default: true dry-run: group: run-mode @@ -164,9 +176,17 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x preprocess: group: src validation: add_deps: original-dataset: tags: _validation +tests: + run_inputs: + - variations_list: + - r2-downloader,mlc,dry-run + - rclone,mlc,dry-run diff --git a/script/get-preprocessed-dataset-nuscenes/README.md b/script/get-preprocessed-dataset-nuscenes/README.md new file mode 100644 index 000000000..5b51c8db4 --- /dev/null +++ b/script/get-preprocessed-dataset-nuscenes/README.md @@ -0,0 +1,80 @@ +# README for get-preprocessed-dataset-nuscenes +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,dataset,nuscenes,preprocessed +``` + +No script specific inputs +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Dataset-src + +- `prebuilt` (default) + +### Dataset-type + +- `calibration` +- `validation` (default) + +### Download-src + +- `mlc` (default) + +### Download-tool + +- `r2-downloader` +- `rclone` (default) + +### Run-mode + +- `dry-run` diff --git a/script/get-preprocessed-dataset-nuscenes/customize.py b/script/get-preprocessed-dataset-nuscenes/customize.py index 5236b5c78..635994ec0 100644 --- a/script/get-preprocessed-dataset-nuscenes/customize.py +++ b/script/get-preprocessed-dataset-nuscenes/customize.py @@ -19,18 +19,26 @@ def postprocess(i): env = i['env'] if is_true(env.get('MLC_TMP_REQUIRE_DOWNLOAD', '')): - env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'] = os.path.join( - env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'], - env['MLC_DATASET_NUSCENES_EXTRACTED_FOLDER_NAME']) - if env.get( - 'MLC_PREPROCESSED_DATASET_NUSCENES_SCENE_LENGTHS_PATH', '') != '': - shutil.copy( - os.path.join( - env['MLC_PREPROCESSED_DATASET_NUSCENES_SCENE_LENGTHS_PATH'], - env['MLC_DATASET_NUSCENES_SCENE_PICKLE_FILENAME']), - os.path.join( - os.path.dirname( - env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'].rstrip("/")), - env['MLC_DATASET_NUSCENES_SCENE_PICKLE_FILENAME'])) + if env.get('MLC_DOWNLOAD_TOOL', '') == "rclone": + env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'] = os.path.join( + env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'], + env['MLC_DATASET_NUSCENES_EXTRACTED_FOLDER_NAME']) + elif env.get('MLC_DOWNLOAD_TOOL', '') == "r2-downloader": + env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'] = os.path.join( + env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'], + "preprocessed", + env['MLC_DATASET_NUSCENES_EXTRACTED_FOLDER_NAME']) + + if env.get('MLC_DOWNLOAD_TOOL', '') == "rclone": + if env.get( + 'MLC_PREPROCESSED_DATASET_NUSCENES_SCENE_LENGTHS_PATH', '') != '': + shutil.copy( + os.path.join( + env['MLC_PREPROCESSED_DATASET_NUSCENES_SCENE_LENGTHS_PATH'], + env['MLC_DATASET_NUSCENES_SCENE_PICKLE_FILENAME']), + os.path.join( + os.path.dirname( + env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'].rstrip("/")), + env['MLC_DATASET_NUSCENES_SCENE_PICKLE_FILENAME'])) return {'return': 0} diff --git a/script/get-preprocessed-dataset-nuscenes/meta.yaml b/script/get-preprocessed-dataset-nuscenes/meta.yaml index 6436574a6..b53494c4a 100644 --- a/script/get-preprocessed-dataset-nuscenes/meta.yaml +++ b/script/get-preprocessed-dataset-nuscenes/meta.yaml @@ -17,42 +17,59 @@ print_env_at_the_end: MLC_PREPROCESSED_DATASET_NUSCENES_PATH: Preprocessed Nuscenes dataset path MLC_PREPROCESSED_DATASET_NUSCENES_ACC_CHECKER_MIN_FILES_PATH: Path containing minimum files for accuracy checker variations: + rclone: + group: download-tool + env: + MLC_DOWNLOAD_TOOL: rclone + add_deps_recursive: + dae: + tags: _rclone + dae_sl: + tags: _rclone + dae_ac: + tags: _rclone + default: true + r2-downloader: + env: + MLC_DOWNLOAD_TOOL: r2-downloader + add_deps_recursive: + dae: + tags: _r2-downloader + dae_sl: + tags: _r2-downloader + dae_ac: + tags: _r2-downloader + group: download-tool + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + prebuilt: + default: true + group: dataset-src + env: + MLC_NUSCENES_DATASET_TYPE: prebuilt validation: default: true group: dataset-type env: MLC_DATASET_NUSCENES_EXTRACTED_FOLDER_NAME: val_3d MLC_DATASET_NUSCENES_TAR_FILENAME: val_3d.tar.gz - MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset/preprocessed/<<>> calibration: group: dataset-type env: MLC_DATASET_NUSCENES_EXTRACTED_FOLDER_NAME: calib_3d MLC_DATASET_NUSCENES_TAR_FILENAME: calib_3d.tar.gz - MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset/preprocessed/<<>> - prebuilt: - default: true - group: dataset-src - env: - MLC_NUSCENES_DATASET_TYPE: prebuilt - MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_PATH - MLC_EXTRACT_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_PATH - MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' mlc: group: download-src default: true prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_config-name.mlc-nuscenes - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - env: - MLC_RCLONE_DRIVE_FOLDER_ID: 17CpM5eU8tjrxh_LpH_BTNTeT37PhzcnC - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' @@ -73,7 +90,30 @@ variations: MLC_DATASET_NUSCENES_SCENE_PICKLE_FILENAME: scene_lengths.pkl MLC_DATASET_NUSCENES_ACC_CHECKER_DEP_FILES_TAR_NAME: nuscenes_min.tar.gz MLC_DATASET_NUSCENES_ACC_REQ_FILE_EXTRACTED_FOLDER_NAME: nuscenes + prebuilt,mlc,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' + prebuilt,mlc: + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_PATH + validation,mlc,rclone: prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + MLC_DOWNLOAD_TOOL: + - rclone + - tags: get,rclone-config,_config-name.mlc-nuscenes + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + MLC_DOWNLOAD_TOOL: + - rclone + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 17CpM5eU8tjrxh_LpH_BTNTeT37PhzcnC - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' @@ -87,10 +127,10 @@ variations: env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_SCENE_LENGTHS_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_SCENE_LENGTHS_PATH - MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset/scene_lengths.pkl + MLC_DOWNLOAD_NUSCENES_SCENE_LENGTHS_URL: mlc-nuscenes:nuscenes_dataset/scene_lengths.pkl update_tags_from_env_with_prefix: _url.: - - MLC_DOWNLOAD_URL + - MLC_DOWNLOAD_NUSCENES_SCENE_LENGTHS_URL - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' @@ -104,29 +144,26 @@ variations: env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_ACC_CHECKER_MIN_FILES_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_ACC_CHECKER_MIN_FILES_PATH - MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset/nuscenes_min.tar.gz + MLC_DOWNLOAD_ACC_CHECKER_MIN_FILES_URL: mlc-nuscenes:nuscenes_dataset/nuscenes_min.tar.gz update_tags_from_env_with_prefix: _url.: - - MLC_DOWNLOAD_URL - rclone: - group: download-tool - add_deps_recursive: - dae: - tags: _rclone - dae_sl: - tags: _rclone - dae_ac: - tags: _rclone - default: true - dry-run: - group: run-mode + - MLC_DOWNLOAD_NUSCENES_SCENE_LENGTHS_URL env: - MLC_DOWNLOAD_MODE: dry - dry-run,rclone: + MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset/preprocessed/<<>> + calibration,mlc,rclone: + MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset/preprocessed/<<>> + validation,mlc,r2-downloader: env: - MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + MLC_DOWNLOAD_URL: https://nuscenes.mlcommons-storage.org/metadata/nuscenes-dataset.uri + calibration,mlc,r2-downloader: + MLC_DOWNLOAD_URL: https://nuscenes.mlcommons-storage.org/metadata/nuscenes-dataset.uri tests: + needs_pat: true run_inputs: - variations_list: - validation,prebuilt,rclone,mlc,dry-run - - calibration,prebuilt,rclone,mlc,dry-run \ No newline at end of file + - calibration,prebuilt,rclone,mlc,dry-run + - validation,prebuilt,r2-downloader,mlc,dry-run + + - calibration,prebuilt,r2-downloader,mlc,dry-run + diff --git a/script/get-preprocessed-dataset-nuscenes/run.sh b/script/get-preprocessed-dataset-nuscenes/run.sh index 16337c44d..1954ef736 100644 --- a/script/get-preprocessed-dataset-nuscenes/run.sh +++ b/script/get-preprocessed-dataset-nuscenes/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" ]]; then +if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" && "$MLC_DOWNLOAD_TOOL" = "rclone" ]]; then cd "${MLC_PREPROCESSED_DATASET_NUSCENES_PATH}" || exit for f in *.tar.gz; do tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } @@ -10,4 +10,16 @@ if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" ]]; t tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } done cd - || exit +fi + +if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" && "$MLC_DOWNLOAD_TOOL" = "r2-downloader" ]]; then + cd "${MLC_PREPROCESSED_DATASET_NUSCENES_PATH}/preprocessed" || exit + for f in *.tar.gz; do + tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + done + cd "${MLC_PREPROCESSED_DATASET_NUSCENES_PATH}" || exit + for f in *.tar.gz; do + tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + done + cd - || exit fi \ No newline at end of file diff --git a/script/get-preprocessed-dataset-openorca/README.md b/script/get-preprocessed-dataset-openorca/README.md index 5899d8c1f..c1f52bc0e 100644 --- a/script/get-preprocessed-dataset-openorca/README.md +++ b/script/get-preprocessed-dataset-openorca/README.md @@ -62,10 +62,19 @@ No script specific inputs - `calibration` (base: mlcommons) - `validation` (default) +### Download-tool + +- `r2-downloader` (default) +- `rclone` + ### Preprocess-step-provider - `nvidia` +### Run-mode + +- `dry-run` + ### Size - `60` diff --git a/script/get-preprocessed-dataset-openorca/customize.py b/script/get-preprocessed-dataset-openorca/customize.py index 4d6af7503..80b61b74d 100644 --- a/script/get-preprocessed-dataset-openorca/customize.py +++ b/script/get-preprocessed-dataset-openorca/customize.py @@ -8,38 +8,39 @@ def preprocess(i): env = i['env'] - if is_true(str(env.get('MLC_DATASET_PREPROCESSED_BY_MLC', ''))): - run_dir = env['MLC_OPENORCA_PREPROCESSED_ROOT'] - run_cmd = '' - env['MLC_DATASET_CALIBRATION_PATH'] = os.path.join( - env['MLC_OPENORCA_PREPROCESSED_ROOT'], - "open_orca_gpt4_tokenized_llama.calibration_1000.pkl.gz") - run_cmd = f"gzip -dkf {env['MLC_DATASET_CALIBRATION_PATH']}" - env['MLC_DATASET_PREPROCESSED_PATH'] = os.path.join( - env['MLC_OPENORCA_PREPROCESSED_ROOT'], - "open_orca_gpt4_tokenized_llama.sampled_24576.pkl.gz") - run_cmd += f" && gzip -dkf {env['MLC_DATASET_PREPROCESSED_PATH']}" - else: - inference_src = env['MLC_MLPERF_INFERENCE_SOURCE'] - run_dir = os.path.join(inference_src, 'language', 'llama2-70b') - model_dir = env['MLC_ML_MODEL_PATH'] - if is_true(env.get('MLC_DATASET_CALIBRATION', '')): - return {'return': 1, 'error': 'No raw preprocessing information is available for openorca calibration. Please use _mlcommons variation to use the MLCommons shared calibration dataset'} - else: + if not env.get('MLC_DOWNLOAD_MODE', '') == "dry": + if is_true(str(env.get('MLC_DATASET_PREPROCESSED_BY_MLC', ''))): + run_dir = env['MLC_OPENORCA_PREPROCESSED_ROOT'] + run_cmd = '' + env['MLC_DATASET_CALIBRATION_PATH'] = os.path.join( + env['MLC_OPENORCA_PREPROCESSED_ROOT'], + "open_orca_gpt4_tokenized_llama.calibration_1000.pkl.gz") + run_cmd = f"gzip -dkf {env['MLC_DATASET_CALIBRATION_PATH']}" env['MLC_DATASET_PREPROCESSED_PATH'] = os.path.join( - os.path.join( - os.getcwd(), - "processed-openorca", - 'open_orca_gpt4_tokenized_llama.sampled_' + - env['MLC_DATASET_SIZE'] + - '.pkl')) - run_cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + ' processorca.py --dataset_pq_path=' + env['MLC_DATASET_OPENORCA_PARQUET'] + ' --model_dir=' + model_dir + \ - ' --seqlen_limit=2048 --export_dir=' + \ - os.path.join(os.getcwd(), "processed-openorca") + \ - ' --num_total_samples=' + env['MLC_DATASET_SIZE'] + env['MLC_OPENORCA_PREPROCESSED_ROOT'], + "open_orca_gpt4_tokenized_llama.sampled_24576.pkl.gz") + run_cmd += f" && gzip -dkf {env['MLC_DATASET_PREPROCESSED_PATH']}" + else: + inference_src = env['MLC_MLPERF_INFERENCE_SOURCE'] + run_dir = os.path.join(inference_src, 'language', 'llama2-70b') + model_dir = env['MLC_ML_MODEL_PATH'] + if is_true(env.get('MLC_DATASET_CALIBRATION', '')): + return {'return': 1, 'error': 'No raw preprocessing information is available for openorca calibration. Please use _mlcommons variation to use the MLCommons shared calibration dataset'} + else: + env['MLC_DATASET_PREPROCESSED_PATH'] = os.path.join( + os.path.join( + os.getcwd(), + "processed-openorca", + 'open_orca_gpt4_tokenized_llama.sampled_' + + env['MLC_DATASET_SIZE'] + + '.pkl')) + run_cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + ' processorca.py --dataset_pq_path=' + env['MLC_DATASET_OPENORCA_PARQUET'] + ' --model_dir=' + model_dir + \ + ' --seqlen_limit=2048 --export_dir=' + \ + os.path.join(os.getcwd(), "processed-openorca") + \ + ' --num_total_samples=' + env['MLC_DATASET_SIZE'] - env['MLC_RUN_DIR'] = run_dir - env['MLC_RUN_CMD'] = run_cmd + env['MLC_RUN_DIR'] = run_dir + env['MLC_RUN_CMD'] = run_cmd return {'return': 0} @@ -47,36 +48,37 @@ def preprocess(i): def postprocess(i): env = i['env'] - if is_true(str(env.get('MLC_DATASET_PREPROCESSED_BY_MLC', ''))): - env['PREPROCESSED_DATA_DIR'] = os.path.dirname( - env['MLC_OPENORCA_PREPROCESSED_ROOT']) - if is_true(env.get('MLC_DATASET_CALIBRATION', '')): - if env.get('MLC_TMP_DATASET_PREPROCESS_STEP_PROVIDER', - '') == "nvidia": - env['MLC_NVIDIA_PREPROCESSED_CALIBRATION_DATASET_PATH'] = os.path.join( - env['MLC_OPENORCA_PREPROCESSED_ROOT'], - "preprocessed_data", - "mlperf_llama2_openorca_calibration_1k") + if not env.get('MLC_DOWNLOAD_MODE', '') == "dry": + if is_true(str(env.get('MLC_DATASET_PREPROCESSED_BY_MLC', ''))): + env['PREPROCESSED_DATA_DIR'] = os.path.dirname( + env['MLC_OPENORCA_PREPROCESSED_ROOT']) + if is_true(env.get('MLC_DATASET_CALIBRATION', '')): + if env.get('MLC_TMP_DATASET_PREPROCESS_STEP_PROVIDER', + '') == "nvidia": + env['MLC_NVIDIA_PREPROCESSED_CALIBRATION_DATASET_PATH'] = os.path.join( + env['MLC_OPENORCA_PREPROCESSED_ROOT'], + "preprocessed_data", + "mlperf_llama2_openorca_calibration_1k") + else: + env['MLC_DATASET_CALIBRATION_PATH'] = os.path.join( + env['MLC_OPENORCA_PREPROCESSED_ROOT'], + "open_orca_gpt4_tokenized_llama.calibration_1000.pkl") + env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_DATASET_CALIBRATION_PATH'] + env['MLC_DATASET_OPENORCA_CALIBRATION_PATH'] = env['MLC_DATASET_CALIBRATION_PATH'] else: - env['MLC_DATASET_CALIBRATION_PATH'] = os.path.join( + env['MLC_DATASET_PREPROCESSED_PATH'] = os.path.join( env['MLC_OPENORCA_PREPROCESSED_ROOT'], - "open_orca_gpt4_tokenized_llama.calibration_1000.pkl") - env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_DATASET_CALIBRATION_PATH'] - env['MLC_DATASET_OPENORCA_CALIBRATION_PATH'] = env['MLC_DATASET_CALIBRATION_PATH'] - else: - env['MLC_DATASET_PREPROCESSED_PATH'] = os.path.join( - env['MLC_OPENORCA_PREPROCESSED_ROOT'], - "open_orca_gpt4_tokenized_llama.sampled_24576.pkl") - if env.get('MLC_TMP_DATASET_PREPROCESS_STEP_PROVIDER', - '') == "nvidia": - env['MLC_DATASET_OPENORCA_NVIDIA_PREPROCESSED_PATH'] = os.path.join( - env['MLC_OPENORCA_PREPROCESSED_ROOT'], - "preprocessed_data") - env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_DATASET_OPENORCA_NVIDIA_PREPROCESSED_PATH'] - else: - env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_DATASET_PREPROCESSED_PATH'] - # The openorca mlc preprocessed dataset is used in nvidia - # implementation for checking accuracy - env['MLC_DATASET_OPENORCA_PREPROCESSED_PATH'] = env['MLC_DATASET_PREPROCESSED_PATH'] + "open_orca_gpt4_tokenized_llama.sampled_24576.pkl") + if env.get('MLC_TMP_DATASET_PREPROCESS_STEP_PROVIDER', + '') == "nvidia": + env['MLC_DATASET_OPENORCA_NVIDIA_PREPROCESSED_PATH'] = os.path.join( + env['MLC_OPENORCA_PREPROCESSED_ROOT'], + "preprocessed_data") + env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_DATASET_OPENORCA_NVIDIA_PREPROCESSED_PATH'] + else: + env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_DATASET_PREPROCESSED_PATH'] + # The openorca mlc preprocessed dataset is used in nvidia + # implementation for checking accuracy + env['MLC_DATASET_OPENORCA_PREPROCESSED_PATH'] = env['MLC_DATASET_PREPROCESSED_PATH'] return {'return': 0} diff --git a/script/get-preprocessed-dataset-openorca/meta.yaml b/script/get-preprocessed-dataset-openorca/meta.yaml index 258accbeb..e27947e1a 100644 --- a/script/get-preprocessed-dataset-openorca/meta.yaml +++ b/script/get-preprocessed-dataset-openorca/meta.yaml @@ -105,24 +105,54 @@ variations: group: size mlc: alias: mlcommons + r2-downloader: + add_deps_recursive: + dae-openorca: + tags: _r2-downloader + group: download-tool + default: true + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + rclone: + add_deps_recursive: + dae-openorca: + tags: _rclone + group: download-tool + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run mlcommons: + default: true deps: - env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_OPENORCA_PREPROCESSED_ROOT MLC_EXTRACT_FINAL_ENV_NAME: MLC_OPENORCA_PREPROCESSED_ROOT MLC_EXTRACT_TO_FOLDER: openorca-preprocessed - MLC_RCLONE_CONFIG_NAME: mlc-inference extra_cache_tags: openorca,preprocessed,dataset force_cache: true names: - dae-openorca - tags: download-and-extract,_rclone + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME update_tags_from_env_with_prefix: _url.: - - MLC_RCLONE_URL + - MLC_DOWNLOAD_URL env: MLC_DATASET_PREPROCESSED_BY_MLC: 'yes' MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/open_orca + r2-downloader,mlcommons: + env: + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/llama-2-70b-open-orca-dataset.uri + rclone,mlcommons: + env: + MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/open_orca + MLC_RCLONE_CONFIG_NAME: mlc-inference size.#: ad: dataset-original: @@ -147,3 +177,8 @@ variations: nvidia,validation: new_env_keys: - MLC_DATASET_OPENORCA_NVIDIA_PREPROCESSED_PATH +tests: + run_inputs: + - variations_list: + - r2-downloader,mlc,dry-run + - rclone,mlc,dry-run diff --git a/script/install-llvm-src/README.md b/script/install-llvm-src/README.md index 7a596672b..baffb8c4f 100644 --- a/script/install-llvm-src/README.md +++ b/script/install-llvm-src/README.md @@ -92,10 +92,20 @@ mlcr install,src,llvm,from.src,src-llvm - `lld` (default) - `no-lld` +### Openmp + +- `no-openmp` +- `openmp` (default) + ### Repo - `repo.#` _(# can be substituted dynamically)_ +### Tests + +- `no-tests` (default) +- `tests` + ### Ungrouped - `branch.#` _(# can be substituted dynamically)_ diff --git a/script/install-llvm-src/customize.py b/script/install-llvm-src/customize.py index 31a914c85..9a82d0c37 100644 --- a/script/install-llvm-src/customize.py +++ b/script/install-llvm-src/customize.py @@ -16,7 +16,17 @@ def preprocess(i): q = '"' if os_info['platform'] == 'windows' else "'" clang_file_name = "clang" - extra_cmake_options = env.get('MLC_LLVM_EXTRA_CMAKE_OPTIONS', '') + + if env.get('+MLC_LLVM_CMAKE_OPTIONS', '') != '': + cmake_options = " ".join(env['+MLC_LLVM_CMAKE_OPTIONS']) + else: + cmake_options = '' + + if not is_true(env.get('MLC_LLVM_PER_TARGET_RUNTIME_DIR')): + cmake_options += " -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF " + + extra_cmake_options = cmake_options + \ + env.get('MLC_LLVM_EXTRA_CMAKE_OPTIONS', '') if env.get('MLC_LLVM_INSTALLED_PATH', '') != '' and os.path.exists( env.get('MLC_LLVM_INSTALLED_PATH')): @@ -24,6 +34,9 @@ def preprocess(i): else: install_prefix = os.path.join(os.getcwd(), "install") + if is_true(env.get('MLC_CLEAN')): + env['MLC_CLEAN_BUILD'] = 'yes' + if os.path.exists(os.path.join(install_prefix, "bin", "clang")) and is_true( env.get('MLC_LLVM_USE_INSTALLED_DIR')): i['run_script_input']['script_name'] = "no-install" # skip install @@ -54,7 +67,7 @@ def preprocess(i): targets_to_build = env.get('MLC_LLVM_TARGETS_TO_BUILD') host_platform = env.get('MLC_HOST_PLATFORM_FLAVOR') if not targets_to_build: - if 'arm64' in host_platform: + if 'arm64' in host_platform or 'aarch64' in host_platform: targets_to_build = 'AArch64' else: targets_to_build = 'X86' diff --git a/script/install-llvm-src/meta.yaml b/script/install-llvm-src/meta.yaml index d145e44c6..1e48dae73 100644 --- a/script/install-llvm-src/meta.yaml +++ b/script/install-llvm-src/meta.yaml @@ -40,6 +40,8 @@ input_mapping: targets: MLC_LLVM_TARGETS_TO_BUILD target_triple: MLC_LLVM_TARGET_TRIPLE extra_options: MLC_LLVM_EXTRA_CMAKE_OPTIONS + per_target_runtime_dir: MLC_LLVM_PER_TARGET_RUNTIME_DIR + clean: MLC_CLEAN env: MLC_GIT_URL: https://github.com/llvm/llvm-project name: Build LLVM compiler from sources (can take >30 min) @@ -78,6 +80,7 @@ variations: env: MLC_GIT_CHECKOUT_TAG: '#' branch.#: + cache_expiration: 5d env: MLC_GIT_CHECKOUT: '#' clang: @@ -86,6 +89,8 @@ variations: +MLC_LLVM_ENABLE_PROJECTS: - clang group: clang + no-clang: + group: clang flang: env: +MLC_LLVM_ENABLE_PROJECTS: @@ -116,11 +121,19 @@ variations: env: +MLC_LLVM_ENABLE_PROJECTS: - lld + +MLC_LLVM_CMAKE_OPTIONS: + - '-DCLANG_DEFAULT_LINKER=lld' group: lld - no-clang: - group: clang no-lld: group: lld + openmp: + default: true + env: + +MLC_LLVM_ENABLE_PROJECTS: + - openmp + group: openmp + no-openmp: + group: openmp debug: env: MLC_LLVM_BUILD_TYPE: debug @@ -134,8 +147,13 @@ variations: - tags: get,generic-sys-util,_crossbuild-essential-arm64 - tags: get,generic-sys-util,_gfortran-12-aarch64-linux-gnu + default_env: + MLC_LLVM_TARGETS_TO_BUILD: AArch64 env: MLC_LLVM_CROSS_COMPILE_FLAGS: "-DCMAKE_SYSTEM_NAME=Linux -DCMAKE_SYSTEM_PROCESSOR=aarch64 -DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-linux-gnu -DCMAKE_Fortran_COMPILER=aarch64-linux-gnu-gfortran-12 -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++" + cross-compile-x86-aarch64,flang: + deps: + - tags: get,generic-sys-util,_gfortran-12-aarch64-linux-gnu for-intel-mlperf-inference-v3.1-bert: adr: conda-package: @@ -276,6 +294,15 @@ variations: - full-history env: MLC_GIT_CHECKOUT_TAG: '#' + no-tests: + group: tests + default: true + env: + MLC_LLVM_CHECK_ALL: '' + tests: + group: tests + env: + MLC_LLVM_CHECK_ALL: check-all docker: skip_run_cmd: true pre_run_cmds: diff --git a/script/install-llvm-src/run.sh b/script/install-llvm-src/run.sh index d71086ce1..f23892265 100644 --- a/script/install-llvm-src/run.sh +++ b/script/install-llvm-src/run.sh @@ -14,7 +14,9 @@ else fi if [[ ${MLC_CLEAN_BUILD} == "yes" ]]; then - rm -rf build + cmd="rm -rf build" + echo "$cmd" + eval "$cmd" fi mkdir -p build @@ -24,14 +26,20 @@ if [ ! -d "${INSTALL_DIR}" ] || [ ${MLC_LLVM_CONDA_ENV} == "yes" ]; then echo "******************************************************" cd build - if [ "${?}" != "0" ]; then exit 1; fi + test $? -eq 0 || exit $? echo "${MLC_LLVM_CMAKE_CMD}" eval "${MLC_LLVM_CMAKE_CMD}" - ninja - if [ "${?}" != "0" ]; then exit 1; fi - ninja install - if [ "${?}" != "0" ]; then exit 1; fi + + cmd="ninja ${MLC_LLVM_CHECK_ALL}" + echo $cmd + eval $cmd + test $? -eq 0 || exit $? + + cmd="ninja install" + echo $cmd + eval $cmd + test $? -eq 0 || exit $? fi diff --git a/script/process-mlperf-accuracy/README.md b/script/process-mlperf-accuracy/README.md new file mode 100644 index 000000000..823fd4a9f --- /dev/null +++ b/script/process-mlperf-accuracy/README.md @@ -0,0 +1,104 @@ +# README for process-mlperf-accuracy +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--rerun` | | | `` | +| `--result_dir` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Category + +- `datacenter` +- `edge` + +### Coco-evaluation-tool + +- `default-pycocotools` (default) +- `nvidia-pycocotools` + +### Dataset + +- `cnndm` +- `cnndm_llama_3` +- `coco2014` +- `cognata_deeplab` +- `cognata_ssd` +- `dataset_llama3` +- `igbh` +- `imagenet` (default) +- `kits19` +- `librispeech` +- `librispeech_whisper` +- `nuscenes` +- `open-orca` +- `openimages` +- `openorca-gsm8k-mbxp` +- `squad` +- `terabyte` +- `waymo` + +### Precision + +- `float16` +- `float32` (default) +- `float64` +- `int16` +- `int32` +- `int64` +- `int8` diff --git a/script/process-mlperf-accuracy/meta.yaml b/script/process-mlperf-accuracy/meta.yaml index 24ff6fdd0..56fe7d5a0 100644 --- a/script/process-mlperf-accuracy/meta.yaml +++ b/script/process-mlperf-accuracy/meta.yaml @@ -94,7 +94,7 @@ variations: - enable_if_env: MLC_MLPERF_IMPLEMENTATION: - intel - tags: get,ml-model,gptj,_fp32,_pytorch + tags: get,ml-model,gptj,_fp32,_pytorch,_r2-downloader env: MLC_DATASET: cnndm group: dataset @@ -235,7 +235,7 @@ variations: skip_if_env: MLC_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST: - 'yes' - tags: get,ml-model,mixtral + tags: get,ml-model,mixtral,_r2-downloader,_mlc env: MLC_DATASET: openorca-gsm8k-mbxp-combined group: dataset @@ -340,7 +340,7 @@ variations: group: dataset cnndm_llama_3: deps: - - tags: get,ml-model,llama3,_hf,_meta-llama/Llama-3.1-8B-Instruct + - tags: get,ml-model,llama3,_8b names: - llama3_1-8b - llama3-8b @@ -353,7 +353,7 @@ variations: group: dataset cnndm_llama_3,edge: deps: - - tags: get,dataset,cnndm,_validation,_edge,_llama3 + - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-edge skip_if_env: @@ -361,7 +361,7 @@ variations: - "yes" cnndm_llama_3,datacenter: deps: - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3 + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-datacenter skip_if_env: diff --git a/script/run-docker-container/README.md b/script/run-docker-container/README.md new file mode 100644 index 000000000..7897f88ff --- /dev/null +++ b/script/run-docker-container/README.md @@ -0,0 +1,102 @@ +# README for run-docker-container +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr run,docker,container +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--all_gpus` | | | `` | +| `--num_gpus` | | | `` | +| `--base` | | | `` | +| `--cache` | | | `` | +| `--mlc_repo` | | | `` | +| `--detached` | | | `` | +| `--device` | | | `` | +| `--docker_image_base` | Alias for base | | `` | +| `--docker_base_image` | Alias for base | | `` | +| `--base_image` | Alias for base | | `` | +| `--keep_detached` | | | `` | +| `--reuse_existing` | | | `no` | +| `--docker_os` | | | `` | +| `--docker_os_version` | | | `` | +| `--os` | Alias for docker_os | | `` | +| `--os_version` | Alias for docker_os_version | | `` | +| `--extra_run_args` | | | `` | +| `--fake_run_option` | | | `` | +| `--gh_token` | | | `` | +| `--image_name` | | | `` | +| `--image_repo` | | | `` | +| `--image_tag` | | | `` | +| `--image_tag_extra` | | | `` | +| `--interactive` | | | `` | +| `--it` | | | `` | +| `--mounts` | | | `` | +| `--pass_user_id` | | | `` | +| `--pass_user_group` | | | `` | +| `--port_maps` | | | `` | +| `--post_run_cmds` | | | `` | +| `--pre_run_cmds` | | | `` | +| `--privileged` | | | `no` | +| `--real_run` | | | `` | +| `--recreate` | | | `` | +| `--rebuild` | Alias for recreate | | `` | +| `--run_cmd` | | | `` | +| `--run_cmd_extra` | | | `` | +| `--save_script` | | | `` | +| `--script_tags` | | | `` | +| `--shm_size` | | | `` | +| `--use_google_dns` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations diff --git a/script/run-docker-container/meta.yaml b/script/run-docker-container/meta.yaml index 4cb3b1ed6..47c6607e2 100644 --- a/script/run-docker-container/meta.yaml +++ b/script/run-docker-container/meta.yaml @@ -50,6 +50,7 @@ input_mapping: port_maps: MLC_DOCKER_PORT_MAPS post_run_cmds: MLC_DOCKER_POST_RUN_COMMANDS pre_run_cmds: MLC_DOCKER_PRE_RUN_COMMANDS + privileged: MLC_DOCKER_PRIVILEGED_MODE real_run: MLC_REAL_RUN recreate: MLC_DOCKER_IMAGE_RECREATE rebuild: MLC_DOCKER_IMAGE_RECREATE