From d89e05fb04c40100f48ca0043b67857f2350a3ac Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 24 Nov 2024 04:54:39 +0530 Subject: [PATCH] autopep python files --- automation/cache/module.py | 16 +- automation/script/module.py | 24 +- automation/script/module_misc.py | 980 ++++++++++-------- .../script/template-ae-python/customize.py | 6 +- automation/script/template-ae-python/main.py | 8 +- automation/script/template-python/main.py | 8 +- automation/script/template-pytorch/main.py | 14 +- automation/script/template/customize.py | 6 +- debug.py | 3 +- script/activate-python-venv/customize.py | 24 +- script/add-custom-nvidia-system/customize.py | 8 +- .../app-image-corner-detection/customize.py | 18 +- .../src/backend_onnxruntime.py | 13 +- .../src/loadgen/harness.py | 3 +- .../src/loadgen/runners.py | 6 +- .../app-mlperf-inference-dummy/customize.py | 29 +- .../nvidia/retinanet.py | 146 ++- .../app-stable-diffusion-onnx-py/process.py | 22 +- script/authenticate-github-cli/customize.py | 6 +- .../customize.py | 20 +- .../customize.py | 19 +- script/compile-program/customize.py | 38 +- script/copy-to-clipboard/code.py | 4 +- script/create-conda-env/customize.py | 12 +- script/create-patch/customize.py | 32 +- script/detect-os/customize.py | 35 +- script/download-and-extract/customize.py | 38 +- script/download-torrent/customize.py | 12 +- script/draw-graph-from-json-data/customize.py | 6 +- .../process-cm-deps.py | 45 +- script/dump-pip-freeze/dump.py | 10 +- script/fail/customize.py | 10 +- script/flash-tinyml-binary/customize.py | 11 +- script/get-cache-dir/customize.py | 6 +- script/get-cmsis_5/customize.py | 8 +- script/get-compiler-flags/customize.py | 12 +- script/get-compiler-rust/customize.py | 10 +- script/get-croissant/customize.py | 7 +- script/get-dataset-cifar10/customize.py | 4 +- script/get-dataset-cnndm/customize.py | 19 +- .../imagenet_helper/__init__.py | 132 ++- .../get-dataset-imagenet-train/customize.py | 27 +- script/get-dataset-kits19/customize.py | 16 +- script/get-dataset-librispeech/customize.py | 13 +- .../customize.py | 9 +- .../filter.py | 11 +- script/get-dataset-openimages/customize.py | 124 ++- script/get-dataset-openorca/customize.py | 10 +- script/get-dataset-squad-vocab/customize.py | 5 +- script/get-dataset-squad/customize.py | 10 +- script/get-gcc/customize.py | 78 +- script/get-generic-python-lib/customize.py | 122 ++- script/get-gh-actions-runner/customize.py | 11 +- script/get-lib-armnn/customize.py | 22 +- script/get-lib-dnnl/customize.py | 21 +- script/get-lib-protobuf/customize.py | 25 +- script/get-lib-qaic-api/customize.py | 28 +- script/get-microtvm/customize.py | 8 +- script/get-ml-model-gptj/customize.py | 51 +- .../get-ml-model-huggingface-zoo/customize.py | 21 +- script/get-ml-model-llama2/customize.py | 18 +- script/get-ml-model-mixtral/customize.py | 6 +- .../download_sparse.py | 2 +- script/get-ml-model-resnet50/customize.py | 15 +- .../customize.py | 29 +- .../customize.py | 6 +- .../customize.py | 4 +- .../customize.py | 8 +- .../get-mlperf-inference-loadgen/customize.py | 16 +- .../customize.py | 10 +- .../customize.py | 10 +- .../customize.py | 10 +- .../get-mlperf-inference-results/customize.py | 16 +- .../customize.py | 13 +- .../customize.py | 105 +- .../get-mlperf-inference-utils/customize.py | 11 +- script/get-mlperf-logging/customize.py | 8 +- script/get-mlperf-power-dev/customize.py | 7 +- .../customize.py | 30 +- script/get-mlperf-tiny-src/customize.py | 44 +- .../customize.py | 11 +- script/get-mlperf-training-src/customize.py | 8 +- script/get-openssl/customize.py | 37 +- .../customize.py | 22 +- .../preprocess.py | 37 +- .../customize.py | 3 +- .../customize.py | 14 +- .../customize.py | 21 +- .../customize.py | 21 +- .../preprocess.py | 39 +- .../customize.py | 28 +- .../customize.py | 57 +- script/get-qaic-apps-sdk/customize.py | 40 +- script/get-qaic-platform-sdk/customize.py | 40 +- script/get-spec-ptd/customize.py | 8 +- script/get-tvm/customize.py | 24 +- script/get-xilinx-sdk/customize.py | 10 +- script/get-zendnn/customize.py | 6 +- script/get-zephyr-sdk/customize.py | 8 +- script/get-zephyr/customize.py | 5 +- script/gui/tests/generate_password.py | 6 +- script/install-apt-package/customize.py | 11 +- script/install-aws-cli/customize.py | 5 +- .../install-cuda-package-manager/customize.py | 6 +- script/install-cuda-prebuilt/customize.py | 24 +- .../install-diffusers-from-src/customize.py | 9 +- script/install-gcc-src/customize.py | 12 +- .../customize.py | 14 +- script/install-gflags-from-src/customize.py | 10 +- script/install-gflags/customize.py | 15 +- script/install-github-cli/customize.py | 3 +- script/install-ipex-from-src/customize.py | 19 +- script/install-llvm-src/customize.py | 33 +- .../customize.py | 6 +- script/install-nccl-libs/customize.py | 6 +- script/install-numactl-from-src/customize.py | 7 +- script/install-onednn-from-src/customize.py | 10 +- .../install-onnxruntime-from-src/customize.py | 7 +- script/install-opencv-from-src/customize.py | 10 +- script/install-openssl/customize.py | 22 +- script/install-python-src/customize.py | 27 +- script/install-pytorch-from-src/customize.py | 17 +- .../customize.py | 5 +- .../install-rapidjson-from-src/customize.py | 8 +- script/install-rocm/customize.py | 10 +- script/install-tensorflow-for-c/customize.py | 21 +- .../install-tensorflow-from-src/customize.py | 47 +- .../install-terraform-from-src/customize.py | 13 +- script/install-tflite-from-src/customize.py | 15 +- .../install-torchvision-from-src/customize.py | 8 +- .../customize.py | 10 +- .../customize.py | 7 +- script/launch-benchmark/tests/debug.py | 8 +- .../plug-prebuilt-cudnn-to-cuda/customize.py | 12 +- .../customize.py | 12 +- script/print-hello-world-py/app.py | 12 +- script/print-hello-world-py/customize.py | 5 +- script/process-ae-users/customize.py | 3 +- script/prune-bert-models/customize.py | 27 +- script/pull-git-repo/customize.py | 9 +- script/push-csv-to-spreadsheet/customize.py | 6 +- script/push-csv-to-spreadsheet/google_api.py | 7 +- .../customize.py | 26 +- .../customize.py | 6 +- .../customize.py | 6 +- .../customize.py | 6 +- .../customize.py | 6 +- .../main.py | 8 +- .../customize.py | 6 +- .../main.py | 8 +- .../customize.py | 6 +- .../customize.py | 6 +- .../main.py | 8 +- .../customize.py | 32 +- .../customize.py | 8 +- .../customize.py | 11 +- .../customize.py | 92 +- .../code.py | 16 +- script/run-mlperf-power-server/customize.py | 19 +- script/run-terraform/customize.py | 19 +- .../customize.py | 33 +- script/set-device-settings-qaic/customize.py | 17 +- script/set-echo-off-win/customize.py | 11 +- script/set-performance-mode/customize.py | 6 +- script/set-sqlite-dir/customize.py | 3 +- script/set-user-limits/customize.py | 6 +- script/tar-my-folder/customize.py | 11 +- script/test-cm-core/customize.py | 3 +- script/test-cm-core/src/script/check.py | 4 +- .../src/script/process_dockerfile.py | 7 +- .../test-cm-core/src/script/process_readme.py | 5 +- script/test-cm-core/src/script/test_deps.py | 26 +- .../test-cm-core/src/script/test_features.py | 20 +- .../test-cm-core/src/script/test_install.py | 9 +- script/test-cm-core/src/test_cm.py | 7 +- script/test-cm-core/src/test_search_speed.py | 13 +- .../src/tutorials/test_tutorial_retinanet.py | 37 +- script/test-cm-script-pipeline/customize.py | 34 +- script/test-cm-scripts/customize.py | 6 +- script/test-debug/customize.py | 33 +- .../customize.py | 6 +- script/test-dummy/customize.py | 6 +- .../customize.py | 5 +- .../customize.py | 7 +- .../customize.py | 28 +- tests/script/process_dockerfile.py | 7 +- tests/script/process_readme.py | 5 +- tests/script/process_tests.py | 8 +- tests/script/test_deps.py | 26 +- tests/script/test_features.py | 25 +- tests/script/test_install.py | 9 +- tests/test_cm.py | 7 +- tests/test_search_speed.py | 13 +- tests/tutorials/test_tutorial_retinanet.py | 37 +- 194 files changed, 2788 insertions(+), 1754 deletions(-) diff --git a/automation/cache/module.py b/automation/cache/module.py index 1e8381f4c6..6720fe5779 100644 --- a/automation/cache/module.py +++ b/automation/cache/module.py @@ -146,14 +146,14 @@ def show(self, i): if len(new_env) > 0: print(' New env:') print( - json.dumps( - new_env, - indent=6, - sort_keys=True).replace( - '{', - '').replace( - '}', - '')) + json.dumps( + new_env, + indent=6, + sort_keys=True).replace( + '{', + '').replace( + '}', + '')) new_state = cached_state.get('new_state', {}) if len(new_state) > 0: diff --git a/automation/script/module.py b/automation/script/module.py index ab768f1ff4..97707b9ced 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -3285,33 +3285,33 @@ def _update_variation_meta_with_dynamic_suffix( for key in variation_meta: value = variation_meta[key] - if type(value) is list: # deps,pre_deps... + if isinstance(value, list): # deps,pre_deps... for item in value: - if type(item) is dict: + if isinstance(item, dict): for item_key in item: item_value = item[item_key] - if type( - item_value) is dict: # env,default_env inside deps + if isinstance( + item_value, dict): # env,default_env inside deps for item_key2 in item_value: item_value[item_key2] = item_value[item_key2].replace( "#", variation_tag_dynamic_suffix) - elif type(item_value) is list: # names for example + elif isinstance(item_value, list): # names for example for i, l_item in enumerate(item_value): - if type(l_item) is str: + if isinstance(l_item, str): item_value[i] = l_item.replace( "#", variation_tag_dynamic_suffix) else: item[item_key] = item[item_key].replace( "#", variation_tag_dynamic_suffix) - elif type(value) is dict: # add_deps, env, .. + elif isinstance(value, dict): # add_deps, env, .. for item in value: item_value = value[item] - if type(item_value) is dict: # deps + if isinstance(item_value, dict): # deps for item_key in item_value: item_value2 = item_value[item_key] - if type( - item_value2) is dict: # env,default_env inside deps + if isinstance( + item_value2, dict): # env,default_env inside deps for item_key2 in item_value2: item_value2[item_key2] = item_value2[item_key2].replace( "#", variation_tag_dynamic_suffix) @@ -3319,9 +3319,9 @@ def _update_variation_meta_with_dynamic_suffix( item_value[item_key] = item_value[item_key].replace( "#", variation_tag_dynamic_suffix) else: - if type(item_value) is list: # lists inside env... + if isinstance(item_value, list): # lists inside env... for i, l_item in enumerate(item_value): - if type(l_item) is str: + if isinstance(l_item, str): item_value[i] = l_item.replace( "#", variation_tag_dynamic_suffix) else: diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index d5c6267f36..44c5bbe183 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -22,13 +22,13 @@ def process_deps(self_module, meta, meta_url, md_script_readme, for kk in [ ('enable_if_env', 'Enable this dependency only if all ENV vars are set'), - ('enable_if_any_env', - 'Enable this dependency only if any of ENV vars are set'), - ('skip_if_env', - 'Skip this dependenecy only if all ENV vars are set'), - ('skip_if_any_env', - 'Skip this dependenecy only if any of ENV vars are set') - ]: + ('enable_if_any_env', + 'Enable this dependency only if any of ENV vars are set'), + ('skip_if_env', + 'Skip this dependenecy only if all ENV vars are set'), + ('skip_if_any_env', + 'Skip this dependenecy only if any of ENV vars are set') + ]: k1 = kk[0] k2 = kk[1] @@ -36,14 +36,14 @@ def process_deps(self_module, meta, meta_url, md_script_readme, conditions = d.get(k1, {}) if len(conditions) > 0: y.append(extra_space + - ' * {}:
\n`{}`'.format(k2, str(conditions))) + ' * {}:
\n`{}`'.format(k2, str(conditions))) if len(names) > 0: y.append( - extra_space + - ' * CM names: `--adr.' + - str(names) + - '...`') + extra_space + + ' * CM names: `--adr.' + + str(names) + + '...`') # Attempt to find related CM scripts r = self_module.cmind.access({'action': 'find', @@ -54,7 +54,7 @@ def process_deps(self_module, meta, meta_url, md_script_readme, if len(lst) == 0: y.append(extra_space + - ' - *Warning: no scripts found*') + ' - *Warning: no scripts found*') else: for s in lst: s_repo_meta = s.repo_meta @@ -85,12 +85,12 @@ def process_deps(self_module, meta, meta_url, md_script_readme, if not skip_if_empty or len(y) > 0: md_script_readme.append( - (extra_space + - ' 1. ' + - x + - 'Read "{}" on other CM scripts' + - z + - x).format(key)) + (extra_space + + ' 1. ' + + x + + 'Read "{}" on other CM scripts' + + z + + x).format(key)) md_script_readme += y ############################################################ @@ -156,8 +156,8 @@ def doc(i): for repo in list_of_repos: parsed_artifact[1] = ( - '', repo) if utils.is_cm_uid(repo) else ( - repo, '') + '', repo) if utils.is_cm_uid(repo) else ( + repo, '') ii['parsed_artifact'] = parsed_artifact r = self_module.search(ii) if r['return'] > 0: @@ -272,7 +272,7 @@ def doc(i): # '', 'Automatically generated README for this automation recipe: **{}**'.format( meta['alias']), - ] + ] md_script.append('## ' + alias) md_script.append('') @@ -364,8 +364,8 @@ def doc(i): # '
', # 'Click to see the summary', '#### Summary', - '' - ] + '' + ] toc_readme.append(x) @@ -426,7 +426,7 @@ def doc(i): cli_all_tags = '`cm run script --tags={}`'.format(','.join(tags)) cli_all_tags3 = '`cm run script --tags={}{} {}`'.format( - ','.join(tags), variation_pointer, input_mapping_pointer) + ','.join(tags), variation_pointer, input_mapping_pointer) x = '* CM CLI with all tags: {}*'.format(cli_all_tags) md_script.append(x) @@ -434,7 +434,7 @@ def doc(i): cli_all_tags_alternative = '`cmr "{}"`'.format(' '.join(tags)) cli_all_tags_alternative3 = '`cmr "{} {}" {}`'.format( - ' '.join(tags), variation_pointer2, input_mapping_pointer) + ' '.join(tags), variation_pointer2, input_mapping_pointer) cli_all_tags_alternative_j = '`cmr "{} {}" {} -j`'.format( ' '.join(tags), variation_pointer, input_mapping_pointer) x = '* CM CLI alternative: {}*'.format(cli_all_tags_alternative) @@ -458,7 +458,7 @@ def doc(i): if default_variation != '': md_script.append( - '* Default variation: *{}*'.format(default_variation)) + '* Default variation: *{}*'.format(default_variation)) if len(version_keys) > 0: md_script.append( @@ -491,7 +491,7 @@ def doc(i): md_script_readme.append('* Output cached? *{}*'.format(str(cache))) md_script_readme.append( - '* See [pipeline of dependencies]({}) on other CM scripts'.format('#dependencies-on-other-cm-scripts')) + '* See [pipeline of dependencies]({}) on other CM scripts'.format('#dependencies-on-other-cm-scripts')) md_script_readme += ['', # '
' @@ -508,36 +508,36 @@ def doc(i): x4 = 'Run this script via Docker (beta)' md_script_readme += [ '', - '---', - '### ' + x1, - '', - '#### ' + x1a, - '', - '* [Install CM](https://access.cknowledge.org/playground/?action=install)', - '* [CM Getting Started Guide](https://github.com/mlcommons/ck/blob/master/docs/getting-started.md)', - '', - '#### ' + x1aa, - '', - '```cm pull repo {}```'.format(repo_alias), - '', - '#### ' + x1b, - '', - '```{}```'.format(cli_help_tags_alternative), - '', - '#### ' + x2, - '', - '{}'.format(cli_all_tags), - '', - '{}'.format(cli_all_tags3), - '', - '*or*', - '', - '{}'.format(cli_all_tags_alternative), - '', - '{}'.format(cli_all_tags_alternative3), - '', + '---', + '### ' + x1, + '', + '#### ' + x1a, + '', + '* [Install CM](https://access.cknowledge.org/playground/?action=install)', + '* [CM Getting Started Guide](https://github.com/mlcommons/ck/blob/master/docs/getting-started.md)', + '', + '#### ' + x1aa, + '', + '```cm pull repo {}```'.format(repo_alias), + '', + '#### ' + x1b, + '', + '```{}```'.format(cli_help_tags_alternative), + '', + '#### ' + x2, + '', + '{}'.format(cli_all_tags), + '', + '{}'.format(cli_all_tags3), + '', + '*or*', + '', + '{}'.format(cli_all_tags_alternative), + '', + '{}'.format(cli_all_tags_alternative3), + '', # '3. {}'.format(cli_uid), - ''] + ''] x = ' and check the [Gettings Started Guide](https://github.com/mlcommons/ck/blob/dev/docs/getting-started.md) for more details.' if len(variation_keys) > 0: @@ -560,7 +560,6 @@ def doc(i): value = input_description[key] desc = value - if isinstance(value, dict): desc = value['desc'] @@ -579,11 +578,10 @@ def doc(i): '**Above CLI flags can be used in the Python CM API as follows:**') md_script_readme.append('') - x = '```python\nr=cm.access({... , "' +key0 +'":...}\n```' + x = '```python\nr=cm.access({... , "' + key0 + '":...}\n```' md_script_readme.append(x) - - md_script_readme += ['#### ' +x3, + md_script_readme += ['#### ' + x3, '', '
', 'Click here to expand this section.', @@ -611,39 +609,37 @@ def doc(i): '', '', - '#### ' +x3a, + '#### ' + x3a, '', - '```cmr "cm gui" --script="' +','.join(tags) +'"```', + '```cmr "cm gui" --script="' + + ','.join(tags) + '"```', '', # 'Use this [online GUI](https://cKnowledge.org/cm-gui/?tags={}) to generate CM CMD.'.format(','.join(tags)), # '', - '#### ' +x4, + '#### ' + x4, '', '{}'.format(cli_all_tags_alternative_docker), '' ] toc_readme.append(x1) - toc_readme.append(' ' +x1a) - toc_readme.append(' ' +x1b) - toc_readme.append(' ' +x2) - toc_readme.append(' ' +x3) - toc_readme.append(' ' +x3a) - toc_readme.append(' ' +x4) + toc_readme.append(' ' + x1a) + toc_readme.append(' ' + x1b) + toc_readme.append(' ' + x2) + toc_readme.append(' ' + x3) + toc_readme.append(' ' + x3a) + toc_readme.append(' ' + x4) x = 'Customization' md_script_readme.append('___') - md_script_readme.append('### ' +x) + md_script_readme.append('### ' + x) md_script_readme.append('') toc_readme.append(x) - - - - if len(variation_keys) >0: - # x = 'Variation groups' - # md_script_readme.append('___') - # md_script_readme.append('### '+x) - # toc_readme.append(x) + if len(variation_keys) > 0: + # x = 'Variation groups' + # md_script_readme.append('___') + # md_script_readme.append('### '+x) + # toc_readme.append(x) variation_groups = {} default_variations = [] @@ -659,7 +655,7 @@ def doc(i): alias = variation.get('alias', '').strip() - if alias !='': + if alias != '': aliases = variation_alias.get(alias, []) if variation_key not in aliases: aliases.append(variation_key) @@ -706,8 +702,8 @@ def doc(i): x = 'Variations' md_script_readme.append('') - md_script_readme.append('#### ' +x) - toc_readme.append(' ' +x) + md_script_readme.append('#### ' + x) + toc_readme.append(' ' + x) variation_groups_order = meta.get('variation_groups_order', []) for variation in sorted(variation_groups): @@ -719,15 +715,15 @@ def doc(i): if not group_key.startswith('*'): md_script_readme.append( - ' * Group "**{}**"'.format(group_key)) + ' * Group "**{}**"'.format(group_key)) else: md_script_readme.append(' * {}'.format(group_key)) md_script_readme += [ '
', - ' Click here to expand this section.', - '' - ] + ' Click here to expand this section.', + '' + ] for variation_key in sorted(variation_groups[group_key]): variation = variations[variation_key] @@ -735,13 +731,13 @@ def doc(i): xmd = variation_md[variation_key] aliases = variation_alias.get(variation_key, []) - aliases2 = ['_' +v for v in aliases] + aliases2 = ['_' + v for v in aliases] - if len(aliases) >0: + if len(aliases) > 0: xmd.append( ' - Aliases: `{}`'.format(','.join(aliases2))) - if len(variation.get('env',{})) >0: + if len(variation.get('env', {})) > 0: xmd.append(' - Environment variables:') for key in variation['env']: xmd.append( @@ -750,19 +746,19 @@ def doc(i): xmd.append(' - Workflow:') for dep in ['deps', 'prehook_deps', - 'posthook_deps', 'post_deps']: + 'posthook_deps', 'post_deps']: process_deps( - self_module, - variation, - meta_url, - xmd, - dep, - ' ', - True, - True) + self_module, + variation, + meta_url, + xmd, + dep, + ' ', + True, + True) for x in xmd: - md_script_readme.append(' ' +x) + md_script_readme.append(' ' + x) md_script_readme.append('') md_script_readme.append('
') @@ -770,51 +766,49 @@ def doc(i): # Check if has invalid_variation_combinations vvc = meta.get('invalid_variation_combinations', []) - if len(vvc) >0: + if len(vvc) > 0: x = 'Unsupported or invalid variation combinations' md_script_readme.append('') - md_script_readme.append('#### ' +x) + md_script_readme.append('#### ' + x) md_script_readme.append('') md_script_readme.append('') md_script_readme.append('') - toc_readme.append(' ' +x) + toc_readme.append(' ' + x) for v in vvc: - vv = ['_' +x for x in v] - md_script_readme.append('* `' +','.join(vv) +'`') - + vv = ['_' + x for x in v] + md_script_readme.append('* `' + ','.join(vv) + '`') - if len(default_variations) >0: + if len(default_variations) > 0: md_script_readme.append('') md_script_readme.append('#### Default variations') md_script_readme.append('') - dv = ['_' +x for x in sorted(default_variations)] + dv = ['_' + x for x in sorted(default_variations)] md_script_readme.append('`{}`'.format(','.join(dv))) # Check if has valid_variation_combinations vvc = meta.get('valid_variation_combinations', []) - if len(vvc) >0: + if len(vvc) > 0: x = 'Valid variation combinations checked by the community' md_script_readme.append('') - md_script_readme.append('#### ' +x) + md_script_readme.append('#### ' + x) md_script_readme.append('') md_script_readme.append('') md_script_readme.append('') - toc_readme.append(' ' +x) + toc_readme.append(' ' + x) for v in vvc: - vv = ['_' +x for x in v] - md_script_readme.append('* `' +','.join(vv) +'`') - + vv = ['_' + x for x in v] + md_script_readme.append('* `' + ','.join(vv) + '`') # Check input flags - if input_mapping and len(input_mapping) >0: + if input_mapping and len(input_mapping) > 0: x = 'Script flags mapped to environment' md_script_readme.append('') - md_script_readme.append('#### ' +x) - toc_readme.append(' ' +x) + md_script_readme.append('#### ' + x) + toc_readme.append(' ' + x) md_script_readme.append('
') md_script_readme.append( @@ -823,17 +817,18 @@ def doc(i): md_script_readme.append('') key0 = '' for key in sorted(input_mapping): - if key0=='': - key0=key + if key0 == '': + key0 = key value = input_mapping[key] - md_script_readme.append('* `--{}=value` → `{}=value`'.format(key, value)) + md_script_readme.append( + '* `--{}=value` → `{}=value`'.format(key, value)) md_script_readme.append('') md_script_readme.append( '**Above CLI flags can be used in the Python CM API as follows:**') md_script_readme.append('') - x = '```python\nr=cm.access({... , "' +key0 +'":...}\n```' + x = '```python\nr=cm.access({... , "' + key0 + '":...}\n```' md_script_readme.append(x) md_script_readme.append('') @@ -845,8 +840,8 @@ def doc(i): x = 'Default environment' # md_script_readme.append('___') - md_script_readme.append('#### ' +x) - toc_readme.append(' ' +x) + md_script_readme.append('#### ' + x) + toc_readme.append(' ' + x) md_script_readme.append('') md_script_readme.append('
') @@ -865,32 +860,25 @@ def doc(i): md_script_readme.append('
') md_script_readme.append('') - - - - - - - if len(version_keys) >0 or default_version!='': + if len(version_keys) > 0 or default_version != '': x = 'Versions' # md_script_readme.append('___') - md_script_readme.append('#### ' +x) + md_script_readme.append('#### ' + x) toc_readme.append(x) - if default_version !='': + if default_version != '': md_script_readme.append( - 'Default version: `{}`'.format(default_version)) + 'Default version: `{}`'.format(default_version)) md_script_readme.append('') - if len(version_keys) >0: + if len(version_keys) > 0: for version in version_keys: md_script_readme.append('* `{}`'.format(version)) - # Add workflow x = 'Dependencies on other CM scripts' md_script_readme += ['___', - '### ' +x, + '### ' + x, ''] toc_readme.append(x) @@ -910,7 +898,7 @@ def doc(i): found_customize = True r = utils.load_txt(path_customize, split=True) - if r['return']>0: + if r['return'] > 0: return r customize = r['string'] @@ -925,21 +913,21 @@ def doc(i): # Ugly attempt to get output env found_postprocess = False for l in customize_l: - # if not found_postprocess: - # if 'def postprocess' in l: - # found_postprocess = True - # else: + # if not found_postprocess: + # if 'def postprocess' in l: + # found_postprocess = True + # else: j = l.find(' env[') - if j >=0: - j1 = l.find(']', j +4) - if j1 >=0: - j2 = l.find('=', j1+1) - if j2 >=0: - key2 = l[j +5:j1].strip() + if j >= 0: + j1 = l.find(']', j + 4) + if j1 >= 0: + j2 = l.find('=', j1 + 1) + if j2 >= 0: + key2 = l[j + 5:j1].strip() key = key2[1:-1] if key.startswith( - 'CM_') and 'TMP' not in key and key not in found_output_env: + 'CM_') and 'TMP' not in key and key not in found_output_env: found_output_env.append(key) process_deps(self_module, meta, meta_url, md_script_readme, 'deps') @@ -948,15 +936,16 @@ def doc(i): y = 'customize.py' if found_customize_preprocess: x = '***' - y = '[' +y +'](' +url +'/' +y +')' - md_script_readme.append((' 1. ' +x +'Run "preprocess" function from {}' +x).format(y)) + y = '[' + y + '](' + url + '/' + y + ')' + md_script_readme.append( + (' 1. ' + x + 'Run "preprocess" function from {}' + x).format(y)) process_deps( - self_module, - meta, - meta_url, - md_script_readme, - 'prehook_deps') + self_module, + meta, + meta_url, + md_script_readme, + 'prehook_deps') # Check scripts files = os.listdir(path) @@ -965,33 +954,35 @@ def doc(i): for f in sorted(files): x = '***' if f.startswith('run') and ( - f.endswith('.sh') or f.endswith('.bat')): - f_url = url +'/' +f + f.endswith('.sh') or f.endswith('.bat')): + f_url = url + '/' + f y.append(' * [{}]({})'.format(f, f_url)) - md_script_readme.append((' 1. ' +x +'Run native script if exists' +x).format(y)) + md_script_readme.append( + (' 1. ' + x + 'Run native script if exists' + x).format(y)) md_script_readme += y process_deps( - self_module, - meta, - meta_url, - md_script_readme, - 'posthook_deps') + self_module, + meta, + meta_url, + md_script_readme, + 'posthook_deps') x = '' y = 'customize.py' if found_customize_postprocess: x = '***' - y = '[' +y +'](' +url +'/' +y +')' - md_script_readme.append((' 1. ' +x +'Run "postrocess" function from {}' +x).format(y)) + y = '[' + y + '](' + url + '/' + y + ')' + md_script_readme.append( + (' 1. ' + x + 'Run "postrocess" function from {}' + x).format(y)) process_deps( - self_module, - meta, - meta_url, - md_script_readme, - 'post_deps') + self_module, + meta, + meta_url, + md_script_readme, + 'post_deps') # md_script_readme.append('
') md_script_readme.append('') @@ -1000,13 +991,13 @@ def doc(i): x = 'Script output' md_script_readme.append('___') - md_script_readme.append('### ' +x) + md_script_readme.append('### ' + x) toc_readme.append(x) md_script_readme.append(cli_all_tags_alternative_j) x = 'New environment keys (filter)' - md_script_readme.append('#### ' +x) + md_script_readme.append('#### ' + x) toc_readme.append(x) md_script_readme.append('') @@ -1030,14 +1021,13 @@ def doc(i): found_output_env_filtered.append(key) x = 'New environment keys auto-detected from customize' - md_script_readme.append('#### ' +x) + md_script_readme.append('#### ' + x) toc_readme.append(x) md_script_readme.append('') for key in sorted(found_output_env_filtered): md_script_readme.append('* `{}`'.format(key)) - # Add maintainers # x = 'Maintainers' # md_script_readme.append('___') @@ -1056,7 +1046,7 @@ def doc(i): prefix = ' ' x2 = x[1:] - x2 = x2.lower().replace(' ', '-').replace(',','') + x2 = x2.lower().replace(' ', '-').replace(',', '') toc_readme_string += prefix + '* [{}](#{})\n'.format(x, x2) # Add to the total list @@ -1069,7 +1059,7 @@ def doc(i): s = s.replace('{{CM_README_TOC}}', toc_readme_string) r = utils.save_txt(path_readme, s) - if r['return']>0: + if r['return'] > 0: return r # Add to Git (if in git) @@ -1080,8 +1070,9 @@ def doc(i): # Recreate TOC with categories toc2 = [] - for category in sorted(toc_category): #, key = lambda x: -toc_category_sort[x]): - toc2.append('### ' +category) + # , key = lambda x: -toc_category_sort[x]): + for category in sorted(toc_category): + toc2.append('### ' + category) toc2.append('') for script in sorted(toc_category[category]): @@ -1093,8 +1084,8 @@ def doc(i): url = urls[script] x = '* [{}]({})'.format(script, url) - if name !='': - x+=' *('+name+')*' + if name != '': + x += ' *(' + name + ')*' toc2.append(x) @@ -1102,12 +1093,12 @@ def doc(i): toc_category_string = '' for category in sorted(toc_category): - category_link = category.lower().replace(' ', '-').replace('/','') + category_link = category.lower().replace(' ', '-').replace('/', '') toc_category_string += '* [{}](#{})\n'.format(category, category_link) # Load template r = utils.load_txt(os.path.join(self_module.path, template_file)) - if r['return']>0: + if r['return'] > 0: return r s = r['string'] @@ -1127,18 +1118,18 @@ def doc(i): output_file = os.path.join(output_dir, list_file) r = utils.save_txt(output_file, s) - if r['return']>0: + if r['return'] > 0: return r out_docs_file = os.path.join( - "..", - "docs", - "scripts", - category, - alias, - "index.md") + "..", + "docs", + "scripts", + category, + alias, + "index.md") r = utils.save_txt(out_docs_file, s) - if r['return']>0: + if r['return'] > 0: return r return {'return': 0} @@ -1153,7 +1144,7 @@ def update_path_for_docker(path, mounts=None, force_path_target=''): path_orig = '' path_target = '' - if path !='': # and (os.path.isfile(path) or os.path.isdir(path)): + if path != '': # and (os.path.isfile(path) or os.path.isdir(path)): path = os.path.abspath(path) path_target = path @@ -1166,9 +1157,10 @@ def update_path_for_docker(path, mounts=None, force_path_target=''): path_target = str(PurePosixPath('/', *x.parts[1:])) if not path_target.startswith('/'): - path_target='/'+path_target + path_target = '/' + path_target - path_target = '/cm-mount'+path_target if force_path_target =='' else force_path_target + path_target = '/cm-mount' + \ + path_target if force_path_target == '' else force_path_target # If file, mount directory if os.path.isfile(path) or not os.path.isdir(path): @@ -1180,7 +1172,7 @@ def update_path_for_docker(path, mounts=None, force_path_target=''): if mounts is not None: to_add = True for y in mounts: - if y.lower() ==x.lower(): + if y.lower() == x.lower(): to_add = False break if to_add: @@ -1190,6 +1182,7 @@ def update_path_for_docker(path, mounts=None, force_path_target=''): ############################################################ + def process_inputs(i): import copy @@ -1204,9 +1197,9 @@ def process_inputs(i): def get_value_using_key_with_dots(d, k): v = None j = k.find('.') - if j >=0: + if j >= 0: k1 = k[:j] - k2 = k[j +1:] + k2 = k[j + 1:] if k1 in d: v = d[k1] @@ -1216,7 +1209,7 @@ def get_value_using_key_with_dots(d, k): else: d = v k = k2 - if type(v) ==dict: + if isinstance(v, dict): v = v.get(k2) else: v = None @@ -1229,19 +1222,19 @@ def get_value_using_key_with_dots(d, k): return v, d, k docker_input_paths = docker_settings.get('input_paths', []) - if len(i_run_cmd) >0: + if len(i_run_cmd) > 0: for k in docker_input_paths: v2, i_run_cmd2, k2 = get_value_using_key_with_dots(i_run_cmd, k) - if v2 !=None: + if v2 is not None: v = i_run_cmd2[k2] path_orig, path_target = update_path_for_docker(v, mounts) - if path_target !='': + if path_target != '': i_run_cmd2[k2] = path_target - return {'return': 0, 'run_cmd':i_run_cmd} + return {'return': 0, 'run_cmd': i_run_cmd} ############################################################ @@ -1289,14 +1282,14 @@ def regenerate_script_cmd(i): if not tags_without_variation: # If no tags without variation, add script alias and UID explicitly - if script_uid!='': - x=script_uid - if script_alias!='': - if x!='': - x=','+x - x = script_alias+x - - if x !='': + if script_uid != '': + x = script_uid + if script_alias != '': + if x != '': + x = ',' + x + x = script_alias + x + + if x != '': run_cmd += ' ' + x + ' ' skip_input_for_fake_run = docker_settings.get( @@ -1314,14 +1307,14 @@ def rebuild_flags(i_run_cmd, fake_run, # Move tags first tags_position = keys.index('tags') del (keys[tags_position]) - keys = ['tags'] +keys + keys = ['tags'] + keys for k in keys: # Assemble long key if dictionary long_key = key_prefix - if long_key!='': - long_key+='.' - long_key+=k + if long_key != '': + long_key += '.' + long_key += k if fake_run and long_key in skip_input_for_fake_run: continue @@ -1330,33 +1323,34 @@ def rebuild_flags(i_run_cmd, fake_run, q = '\\"' if long_key in add_quotes_to_keys else '' - if type(v) ==dict: + if isinstance(v, dict): run_cmd += rebuild_flags(v, - fake_run, - skip_input_for_fake_run, - add_quotes_to_keys, - long_key) - elif type(v) ==list: + fake_run, + skip_input_for_fake_run, + add_quotes_to_keys, + long_key) + elif isinstance(v, list): x = '' for vv in v: if x != '': - x+=',' - x+=q+str(vv)+q - run_cmd += ' --'+long_key+',=' + x + x += ',' + x += q + str(vv) + q + run_cmd += ' --' + long_key + ',=' + x else: - run_cmd += ' --'+long_key+'='+q+str(v)+q + run_cmd += ' --' + long_key + '=' + q + str(v) + q return run_cmd run_cmd += rebuild_flags(i_run_cmd, - fake_run, - skip_input_for_fake_run, - add_quotes_to_keys, - '') + fake_run, + skip_input_for_fake_run, + add_quotes_to_keys, + '') - run_cmd = docker_run_cmd_prefix + ' && ' + run_cmd if docker_run_cmd_prefix != '' else run_cmd + run_cmd = docker_run_cmd_prefix + ' && ' + \ + run_cmd if docker_run_cmd_prefix != '' else run_cmd - return {'return': 0, 'run_cmd_string':run_cmd} + return {'return': 0, 'run_cmd_string': run_cmd} ############################################################ @@ -1375,9 +1369,9 @@ def aux_search(i): parsed_artifact = inp.get('parsed_artifact', []) - if len(parsed_artifact) <1: - parsed_artifact = [('', ''), ('','')] - elif len(parsed_artifact) <2: + if len(parsed_artifact) < 1: + parsed_artifact = [('', ''), ('', '')] + elif len(parsed_artifact) < 2: parsed_artifact.append(('', '')) else: repos = parsed_artifact[1][0] @@ -1385,23 +1379,25 @@ def aux_search(i): list_of_repos = repos.split(',') if ',' in repos else [repos] ii = utils.sub_input( - inp, - self_module.cmind.cfg['artifact_keys'] + - ['tags']) + inp, + self_module.cmind.cfg['artifact_keys'] + + ['tags']) ii['out'] = None # Search for automations in repos lst = [] for repo in list_of_repos: - parsed_artifact[1] = ('', repo) if utils.is_cm_uid(repo) else (repo,'') + parsed_artifact[1] = ( + '', repo) if utils.is_cm_uid(repo) else ( + repo, '') ii['parsed_artifact'] = parsed_artifact r = self_module.search(ii) - if r['return']>0: + if r['return'] > 0: return r lst += r['list'] - return {'return': 0, 'list':lst} + return {'return': 0, 'list': lst} ############################################################ @@ -1429,10 +1425,14 @@ def dockerfile(i): # Check simplified CMD: cm docker script "python app image-classification onnx" # If artifact has spaces, treat them as tags! self_module = i['self_module'] - self_module.cmind.access({'action': 'detect_tags_in_artifact', 'automation':'utils', 'input':i}) + self_module.cmind.access( + {'action': 'detect_tags_in_artifact', 'automation': 'utils', 'input': i}) # Prepare "clean" input to replicate command - r = self_module.cmind.access({'action': 'prune_input', 'automation':'utils', 'input':i, 'extra_keys_starts_with':['docker_']}) + r = self_module.cmind.access({'action': 'prune_input', + 'automation': 'utils', + 'input': i, + 'extra_keys_starts_with': ['docker_']}) i_run_cmd_arc = r['new_input'] cur_dir = os.getcwd() @@ -1443,13 +1443,13 @@ def dockerfile(i): # Search for script(s) r = aux_search({'self_module': self_module, 'input': i}) - if r['return']>0: + if r['return'] > 0: return r lst = r['list'] - if len(lst) ==0: - return {'return': 1, 'error':'no scripts were found'} + if len(lst) == 0: + return {'return': 1, 'error': 'no scripts were found'} # if i.get('cmd'): @@ -1463,10 +1463,6 @@ def dockerfile(i): # # run_cmd = i.get('docker_run_cmd_prefix') + ' && ' + run_cmd if i.get('docker_run_cmd_prefix') else run_cmd - - - - env = i.get('env', {}) state = i.get('state', {}) const = i.get('const', {}) @@ -1476,9 +1472,9 @@ def dockerfile(i): dockerfile_env = i.get('dockerfile_env', {}) tags_split = i.get('tags', '').split(",") - variation_tags = [t[1:] for t in tags_split if t.startswith("_") ] + variation_tags = [t[1:] for t in tags_split if t.startswith("_")] - for artifact in sorted(lst, key = lambda x: x.meta.get('alias', '')): + for artifact in sorted(lst, key=lambda x: x.meta.get('alias', '')): meta = artifact.meta @@ -1493,7 +1489,7 @@ def dockerfile(i): verbose = i.get('v', False) show_time = i.get('show_time', False) - run_state = {'deps': [], 'fake_deps':[], 'parent': None} + run_state = {'deps': [], 'fake_deps': [], 'parent': None} run_state['script_id'] = script_alias + "," + script_uid run_state['script_variation_tags'] = variation_tags variations = meta.get('variations', {}) @@ -1502,11 +1498,42 @@ def dockerfile(i): state['docker'] = docker_settings add_deps_recursive = i.get('add_deps_recursive', {}) - r = script_automation.update_state_from_meta(meta, env, state, const, const_state, deps= [], post_deps = [], prehook_deps = [], posthook_deps = [], new_env_keys = [], new_state_keys = [], run_state=run_state, i = i) + r = script_automation.update_state_from_meta( + meta, + env, + state, + const, + const_state, + deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys=[], + new_state_keys=[], + run_state=run_state, + i=i) if r['return'] > 0: return r - r = script_automation._update_state_from_variations(i, meta, variation_tags, variations, env, state, const, const_state, deps= [], post_deps = [], prehook_deps = [], posthook_deps = [], new_env_keys_from_meta = [], new_state_keys_from_meta = [], add_deps_recursive = add_deps_recursive, run_state = run_state, recursion_spaces='', verbose = False) + r = script_automation._update_state_from_variations( + i, + meta, + variation_tags, + variations, + env, + state, + const, + const_state, + deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys_from_meta=[], + new_state_keys_from_meta=[], + add_deps_recursive=add_deps_recursive, + run_state=run_state, + recursion_spaces='', + verbose=False) if r['return'] > 0: return r @@ -1515,7 +1542,7 @@ def dockerfile(i): dockerfile_env['CM_RUN_STATE_DOCKER'] = True if not docker_settings.get('run', True) and not i.get( - 'docker_run_override', False): + 'docker_run_override', False): print("docker.run set to False in _cm.json") continue '''run_config_path = os.path.join(script_path,'run_config.yml') @@ -1534,26 +1561,39 @@ def dockerfile(i): deps = docker_settings.get('build_deps', []) if deps: r = script_automation._run_deps( - deps, - [], - env, - {}, - {}, - {}, - {}, - '', - [], - '', - False, - '', - verbose, - show_time, - ' ', - run_state) + deps, + [], + env, + {}, + {}, + {}, + {}, + '', + [], + '', + False, + '', + verbose, + show_time, + ' ', + run_state) if r['return'] > 0: return r # For updating meta from update_meta_if_env - r = script_automation.update_state_from_meta(meta, env, state, const, const_state, deps= [], post_deps = [], prehook_deps = [], posthook_deps = [], new_env_keys = [], new_state_keys = [], run_state=run_state, i = i) + r = script_automation.update_state_from_meta( + meta, + env, + state, + const, + const_state, + deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys=[], + new_state_keys=[], + run_state=run_state, + i=i) if r['return'] > 0: return r docker_settings = state['docker'] @@ -1567,14 +1607,14 @@ def dockerfile(i): r = process_inputs({'run_cmd_arc': i_run_cmd_arc, 'docker_settings': docker_settings, 'mounts': []}) - if r['return']>0: + if r['return'] > 0: return r i_run_cmd = r['run_cmd'] docker_run_cmd_prefix = i.get( - 'docker_run_cmd_prefix', docker_settings.get( - 'run_cmd_prefix', '')) + 'docker_run_cmd_prefix', docker_settings.get( + 'run_cmd_prefix', '')) r = regenerate_script_cmd({'script_uid': script_uid, 'script_alias': script_alias, @@ -1583,83 +1623,85 @@ def dockerfile(i): 'fake_run': True, 'docker_settings': docker_settings, 'docker_run_cmd_prefix': docker_run_cmd_prefix}) - if r['return']>0: + if r['return'] > 0: return r run_cmd = r['run_cmd_string'] cm_repo = i.get( - 'docker_cm_repo', - docker_settings.get( - 'cm_repo', - 'mlcommons@cm4mlops')) + 'docker_cm_repo', + docker_settings.get( + 'cm_repo', + 'mlcommons@cm4mlops')) cm_repo_branch = i.get( - 'docker_cm_repo_branch', - docker_settings.get( - 'cm_repo_branch', - 'mlperf-inference')) + 'docker_cm_repo_branch', + docker_settings.get( + 'cm_repo_branch', + 'mlperf-inference')) cm_repo_flags = i.get( - 'docker_cm_repo_flags', - docker_settings.get( - 'cm_repo_flags', - '')) + 'docker_cm_repo_flags', + docker_settings.get( + 'cm_repo_flags', + '')) docker_base_image = i.get( - 'docker_base_image', - docker_settings.get('base_image')) + 'docker_base_image', + docker_settings.get('base_image')) docker_os = i.get( - 'docker_os', docker_settings.get( - 'docker_os', 'ubuntu')) + 'docker_os', docker_settings.get( + 'docker_os', 'ubuntu')) docker_os_version = i.get( - 'docker_os_version', docker_settings.get( - 'docker_os_version', '22.04')) + 'docker_os_version', docker_settings.get( + 'docker_os_version', '22.04')) docker_cm_repos = i.get( - 'docker_cm_repos', - docker_settings.get( - 'cm_repos', - '')) + 'docker_cm_repos', + docker_settings.get( + 'cm_repos', + '')) docker_skip_cm_sys_upgrade = i.get( - 'docker_skip_cm_sys_upgrade', docker_settings.get( - 'skip_cm_sys_upgrade', '')) + 'docker_skip_cm_sys_upgrade', docker_settings.get( + 'skip_cm_sys_upgrade', '')) docker_extra_sys_deps = i.get('docker_extra_sys_deps', '') if not docker_base_image: - dockerfilename_suffix = docker_os + '_'+docker_os_version + dockerfilename_suffix = docker_os + '_' + docker_os_version else: if os.name == 'nt': - dockerfilename_suffix = docker_base_image.replace('/', '-').replace(':', '-') + dockerfilename_suffix = docker_base_image.replace( + '/', '-').replace(':', '-') else: dockerfilename_suffix = docker_base_image.split("/") dockerfilename_suffix = dockerfilename_suffix[len( dockerfilename_suffix) - 1] fake_run_deps = i.get( - 'fake_run_deps', docker_settings.get( - 'fake_run_deps', False)) + 'fake_run_deps', docker_settings.get( + 'fake_run_deps', False)) docker_run_final_cmds = docker_settings.get( 'docker_run_final_cmds', []) r = check_gh_token(i, docker_settings, quiet) - if r['return'] >0 : + if r['return'] > 0: return r gh_token = r['gh_token'] i['docker_gh_token'] = gh_token # To pass to docker function if needed - if i.get('docker_real_run', docker_settings.get('docker_real_run', False)): + if i.get('docker_real_run', docker_settings.get( + 'docker_real_run', False)): fake_run_option = " " fake_run_deps = False else: fake_run_option = " --fake_run" docker_copy_files = i.get( - 'docker_copy_files', - docker_settings.get( - 'copy_files', - [])) + 'docker_copy_files', + docker_settings.get( + 'copy_files', + [])) env['CM_DOCKER_PRE_RUN_COMMANDS'] = docker_run_final_cmds @@ -1667,7 +1709,11 @@ def dockerfile(i): if docker_path == '': docker_path = script_path - dockerfile_path = os.path.join(docker_path, 'dockerfiles', dockerfilename_suffix + '.Dockerfile') + dockerfile_path = os.path.join( + docker_path, + 'dockerfiles', + dockerfilename_suffix + + '.Dockerfile') if i.get('print_deps'): cm_input = {'action': 'run', @@ -1683,7 +1729,7 @@ def dockerfile(i): if r['return'] > 0: return r print_deps = r['new_state']['print_deps'] - comments = ["#RUN " + dep for dep in print_deps ] + comments = ["#RUN " + dep for dep in print_deps] comments.append("") comments.append("# Run CM workflow") else: @@ -1738,6 +1784,7 @@ def dockerfile(i): # we mount the main folder of the CM cache entry in case any file/folder # in that cache entry is needed inside the container + def get_host_path(value): path_split = value.split(os.sep) if len(path_split) == 1: @@ -1747,7 +1794,7 @@ def get_host_path(value): if "cache" in path_split and "local": repo_entry_index = path_split.index("local") if len(path_split) >= repo_entry_index + 3: - return os.sep.join(path_split[0:repo_entry_index +3]) + return os.sep.join(path_split[0:repo_entry_index + 3]) return value @@ -1765,10 +1812,11 @@ def get_container_path(value): new_value = '' if "cache" in path_split and "local" in path_split: - new_path_split = ["", "home", "cmuser", "CM", "repos" ] + new_path_split = ["", "home", "cmuser", "CM", "repos"] repo_entry_index = path_split.index("local") if len(path_split) >= repo_entry_index + 3: - new_path_split1 = new_path_split + path_split[repo_entry_index:repo_entry_index +3] + new_path_split1 = new_path_split + \ + path_split[repo_entry_index:repo_entry_index + 3] new_path_split2 = new_path_split + path_split[repo_entry_index:] return "/".join(new_path_split1), "/".join(new_path_split2) else: @@ -1804,7 +1852,7 @@ def docker(i): self_module = i['self_module'] - if type(i.get('docker', None)) == dict: + if isinstance(i.get('docker', None), dict): # Grigori started cleaning and refactoring this code on 20240929 # # 1. use --docker dictionary instead of --docker_{keys} @@ -1817,9 +1865,9 @@ def docker(i): quiet = i.get('quiet', False) detached = i.get('docker_detached', '') - if detached =='': + if detached == '': detached = i.get('docker_dt', '') - if detached =='': + if detached == '': detached = 'no' interactive = i.get('docker_interactive', '') @@ -1831,16 +1879,21 @@ def docker(i): # Check simplified CMD: cm docker script "python app image-classification onnx" # If artifact has spaces, treat them as tags! - self_module.cmind.access({'action': 'detect_tags_in_artifact', 'automation':'utils', 'input':i}) + self_module.cmind.access( + {'action': 'detect_tags_in_artifact', 'automation': 'utils', 'input': i}) # CAREFUL -> artifacts and parsed_artifacts are not supported in input # (and should not be?) if 'artifacts' in i: - del(i['artifacts']) - if 'parsed_artifacts' in i: del(i['parsed_artifacts']) + del (i['artifacts']) + if 'parsed_artifacts' in i: + del (i['parsed_artifacts']) # Prepare "clean" input to replicate command - r = self_module.cmind.access({'action': 'prune_input', 'automation':'utils', 'input':i, 'extra_keys_starts_with':['docker_']}) + r = self_module.cmind.access({'action': 'prune_input', + 'automation': 'utils', + 'input': i, + 'extra_keys_starts_with': ['docker_']}) i_run_cmd_arc = r['new_input'] env = i.get('env', {}) @@ -1859,7 +1912,8 @@ def docker(i): if docker_cfg != '' or docker_cfg_uid != '': # Check if docker_cfg is turned on but not selected - if type(docker_cfg) == bool or str(docker_cfg).lower() in ['true', 'yes']: + if isinstance(docker_cfg, bool) or str( + docker_cfg).lower() in ['true', 'yes']: docker_cfg = '' r = self_module.cmind.access({'action': 'select_cfg', @@ -1870,7 +1924,8 @@ def docker(i): 'uid': docker_cfg_uid}) if r['return'] > 0: if r['return'] == 16: - return {'return': 1, 'error':'Docker configuration {} was not found'.format(docker_cfg)} + return {'return': 1, 'error': 'Docker configuration {} was not found'.format( + docker_cfg)} return r selection = r['selection'] @@ -1883,8 +1938,8 @@ def docker(i): # Run dockerfile if not noregenerate_docker_file: r = utils.call_internal_module( - self_module, __file__, 'module_misc', 'dockerfile', i) - if r['return']>0: + self_module, __file__, 'module_misc', 'dockerfile', i) + if r['return'] > 0: return r # Save current directory @@ -1894,13 +1949,13 @@ def docker(i): # Search for script(s) r = aux_search({'self_module': self_module, 'input': i}) - if r['return']>0: + if r['return'] > 0: return r lst = r['list'] - if len(lst) ==0: - return {'return': 1, 'error':'no scripts were found'} + if len(lst) == 0: + return {'return': 1, 'error': 'no scripts were found'} env['CM_RUN_STATE_DOCKER'] = False script_automation = i['self_module'] @@ -1909,7 +1964,7 @@ def docker(i): const_state = i.get('const_state', {}) tags_split = i.get('tags', '').split(",") - variation_tags = [t[1:] for t in tags_split if t.startswith("_") ] + variation_tags = [t[1:] for t in tags_split if t.startswith("_")] docker_cache = i.get('docker_cache', "yes") if docker_cache in ["no", False, "False"]: @@ -1922,17 +1977,18 @@ def docker(i): # Host system needs to have docker r = self_module.cmind.access({'action': 'run', - 'automation': 'script', + 'automation': 'script', 'tags': "get,docker"}) if r['return'] > 0: return r - for artifact in sorted(lst, key = lambda x: x.meta.get('alias', '')): + for artifact in sorted(lst, key=lambda x: x.meta.get('alias', '')): meta = artifact.meta if i.get('help', False): - return utils.call_internal_module(self_module, __file__, 'module_help', 'print_help', {'meta': meta, 'path':artifact.path}) + return utils.call_internal_module(self_module, __file__, 'module_help', 'print_help', { + 'meta': meta, 'path': artifact.path}) script_path = artifact.path @@ -1957,23 +2013,54 @@ def docker(i): docker_settings = meta.get('docker', {}) state['docker'] = docker_settings # Todo: Support state, const and add_deps_recursive - run_state = {'deps': [], 'fake_deps':[], 'parent': None} + run_state = {'deps': [], 'fake_deps': [], 'parent': None} run_state['script_id'] = script_alias + "," + script_uid run_state['script_variation_tags'] = variation_tags add_deps_recursive = i.get('add_deps_recursive', {}) - r = script_automation.update_state_from_meta(meta, env, state, const, const_state, deps= [], post_deps = [], prehook_deps = [], posthook_deps = [], new_env_keys = [], new_state_keys = [], run_state=run_state, i = i) + r = script_automation.update_state_from_meta( + meta, + env, + state, + const, + const_state, + deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys=[], + new_state_keys=[], + run_state=run_state, + i=i) if r['return'] > 0: return r - r = script_automation._update_state_from_variations(i, meta, variation_tags, variations, env, state, const, const_state, deps= [], post_deps = [], prehook_deps = [], posthook_deps = [], new_env_keys_from_meta = [], new_state_keys_from_meta = [], add_deps_recursive = add_deps_recursive, run_state = run_state, recursion_spaces='', verbose = False) + r = script_automation._update_state_from_variations( + i, + meta, + variation_tags, + variations, + env, + state, + const, + const_state, + deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys_from_meta=[], + new_state_keys_from_meta=[], + add_deps_recursive=add_deps_recursive, + run_state=run_state, + recursion_spaces='', + verbose=False) if r['return'] > 0: return r docker_settings = state['docker'] if not docker_settings.get('run', True) and not i.get( - 'docker_run_override', False): + 'docker_run_override', False): print("docker.run set to False in _cm.json") continue ''' @@ -1986,50 +2073,63 @@ def docker(i): r = process_inputs({'run_cmd_arc': i_run_cmd_arc, 'docker_settings': docker_settings, 'mounts': mounts}) - if r['return']>0: + if r['return'] > 0: return r i_run_cmd = r['run_cmd'] # Check if need to mount home directory current_path_target = '/cm-mount/current' - if docker_settings.get('mount_current_dir','') =='yes': + if docker_settings.get('mount_current_dir', '') == 'yes': update_path_for_docker( - '.', mounts, force_path_target=current_path_target) + '.', mounts, force_path_target=current_path_target) _os = i.get('docker_os', docker_settings.get('os', 'ubuntu')) version = i.get( - 'docker_os_version', - docker_settings.get( - 'os_version', - '22.04')) + 'docker_os_version', + docker_settings.get( + 'os_version', + '22.04')) build_deps = docker_settings.get('deps', []) deps = docker_settings.get('deps', []) deps = build_deps + deps if deps: r = script_automation._run_deps( - deps, - [], - env, - {}, - {}, - {}, - {}, - '', - [], - '', - False, - '', - verbose, - show_time, - ' ', - run_state) + deps, + [], + env, + {}, + {}, + {}, + {}, + '', + [], + '', + False, + '', + verbose, + show_time, + ' ', + run_state) if r['return'] > 0: return r # For updating meta from update_meta_if_env - r = script_automation.update_state_from_meta(meta, env, state, const, const_state, deps= [], post_deps = [], prehook_deps = [], posthook_deps = [], new_env_keys = [], new_state_keys = [], run_state=run_state, i = i) + r = script_automation.update_state_from_meta( + meta, + env, + state, + const, + const_state, + deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys=[], + new_state_keys=[], + run_state=run_state, + i=i) if r['return'] > 0: return r @@ -2057,15 +2157,18 @@ def docker(i): env[docker_input_mapping[c_input]] = i[c_input] # del(i[c_input]) - container_env_string = '' # env keys corresponding to container mounts are explicitly passed to the container run cmd + # env keys corresponding to container mounts are explicitly passed to + # the container run cmd + container_env_string = '' for index in range(len(mounts)): mount = mounts[index] # Since windows may have 2 :, we search from the right j = mount.rfind(':') - if j >0: - mount_parts = [mount[:j], mount[j +1:]] + if j > 0: + mount_parts = [mount[:j], mount[j + 1:]] else: - return {'return': 1, 'error': 'Can\'t find separator : in a mount string: {}'.format(mount)} + return { + 'return': 1, 'error': 'Can\'t find separator : in a mount string: {}'.format(mount)} # mount_parts = mount.split(":") # if len(mount_parts) != 2: @@ -2085,7 +2188,7 @@ def docker(i): if tmp_value in env: host_env_key = tmp_value new_host_mount = get_host_path(env[tmp_value]) - else: # we skip those mounts + else: # we skip those mounts mounts[index] = None skip = True break @@ -2099,7 +2202,7 @@ def docker(i): env[tmp_value]) container_env_key = new_container_mount_env # container_env_string += " --env.{}={} ".format(tmp_value, new_container_mount_env) - else: # we skip those mounts + else: # we skip those mounts mounts[index] = None skip = True break @@ -2108,7 +2211,7 @@ def docker(i): if skip: continue - mounts[index] = new_host_mount +":" +new_container_mount + mounts[index] = new_host_mount + ":" + new_container_mount if host_env_key: container_env_string += " --env.{}={} ".format( host_env_key, container_env_key) @@ -2123,7 +2226,18 @@ def docker(i): mount_string = "" if len(mounts) == 0 else ",".join(mounts) # check for proxy settings and pass onto the docker - proxy_keys = ["ftp_proxy", "FTP_PROXY", "http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "no_proxy", "NO_PROXY", "socks_proxy", "SOCKS_PROXY", "GH_TOKEN" ] + proxy_keys = [ + "ftp_proxy", + "FTP_PROXY", + "http_proxy", + "HTTP_PROXY", + "https_proxy", + "HTTPS_PROXY", + "no_proxy", + "NO_PROXY", + "socks_proxy", + "SOCKS_PROXY", + "GH_TOKEN"] if env.get('+ CM_DOCKER_BUILD_ARGS', []) == []: env['+ CM_DOCKER_BUILD_ARGS'] = [] @@ -2136,63 +2250,72 @@ def docker(i): "{}={}".format(key, value)) docker_use_host_group_id = i.get( - 'docker_use_host_group_id', - docker_settings.get('use_host_group_id')) + 'docker_use_host_group_id', + docker_settings.get('use_host_group_id')) if str(docker_use_host_group_id).lower() not in [ - 'false', 'no', '0'] and os.name != 'nt': + 'false', 'no', '0'] and os.name != 'nt': env['+ CM_DOCKER_BUILD_ARGS'].append( "{}={}".format('GID', '\\" $(id -g $USER) \\"')) docker_use_host_user_id = i.get( - 'docker_use_host_user_id', - docker_settings.get('use_host_user_id')) + 'docker_use_host_user_id', + docker_settings.get('use_host_user_id')) if str(docker_use_host_user_id).lower() not in [ - 'false', 'no', '0'] and os.name != 'nt': + 'false', 'no', '0'] and os.name != 'nt': env['+ CM_DOCKER_BUILD_ARGS'].append( "{}={}".format('UID', '\\" $(id -u $USER) \\"')) docker_base_image = i.get( - 'docker_base_image', - docker_settings.get('base_image')) + 'docker_base_image', + docker_settings.get('base_image')) docker_os = i.get('docker_os', docker_settings.get('os', 'ubuntu')) docker_os_version = i.get( - 'docker_os_version', docker_settings.get( - 'os_version', '22.04')) + 'docker_os_version', docker_settings.get( + 'os_version', '22.04')) image_tag_extra = i.get( - 'docker_image_tag_extra', - docker_settings.get( - 'image_tag_extra', - '-latest')) + 'docker_image_tag_extra', + docker_settings.get( + 'image_tag_extra', + '-latest')) if not docker_base_image: - dockerfilename_suffix = docker_os + '_'+docker_os_version + dockerfilename_suffix = docker_os + '_' + docker_os_version else: if os.name == 'nt': - dockerfilename_suffix = docker_base_image.replace('/', '-').replace(':', '-') + dockerfilename_suffix = docker_base_image.replace( + '/', '-').replace(':', '-') else: dockerfilename_suffix = docker_base_image.split("/") dockerfilename_suffix = dockerfilename_suffix[len( dockerfilename_suffix) - 1] - - cm_repo = i.get('docker_cm_repo', docker_settings.get('cm_repo', 'mlcommons@cm4mlops')) + cm_repo = i.get( + 'docker_cm_repo', + docker_settings.get( + 'cm_repo', + 'mlcommons@cm4mlops')) docker_path = i.get('docker_path', '').strip() if docker_path == '': docker_path = script_path - dockerfile_path = os.path.join(docker_path, 'dockerfiles', dockerfilename_suffix + '.Dockerfile') + dockerfile_path = os.path.join( + docker_path, + 'dockerfiles', + dockerfilename_suffix + + '.Dockerfile') # Skips docker run cmd and gives an interactive shell to the user docker_skip_run_cmd = i.get( - 'docker_skip_run_cmd', docker_settings.get( - 'skip_run_cmd', False)) + 'docker_skip_run_cmd', docker_settings.get( + 'skip_run_cmd', False)) - docker_pre_run_cmds = i.get('docker_pre_run_cmds', []) + docker_settings.get('pre_run_cmds', []) + docker_pre_run_cmds = i.get( + 'docker_pre_run_cmds', []) + docker_settings.get('pre_run_cmds', []) docker_run_cmd_prefix = i.get( - 'docker_run_cmd_prefix', docker_settings.get( - 'run_cmd_prefix', '')) + 'docker_run_cmd_prefix', docker_settings.get( + 'run_cmd_prefix', '')) all_gpus = i.get('docker_all_gpus', docker_settings.get('all_gpus')) @@ -2201,49 +2324,49 @@ def docker(i): device = i.get('docker_device', docker_settings.get('device')) image_name = i.get( - 'docker_image_name', - docker_settings.get( - 'image_name', - '')) + 'docker_image_name', + docker_settings.get( + 'image_name', + '')) r = check_gh_token(i, docker_settings, quiet) - if r['return'] >0 : + if r['return'] > 0: return r gh_token = r['gh_token'] port_maps = i.get( - 'docker_port_maps', - docker_settings.get( - 'port_maps', - [])) + 'docker_port_maps', + docker_settings.get( + 'port_maps', + [])) shm_size = i.get( - 'docker_shm_size', - docker_settings.get( - 'shm_size', - '')) + 'docker_shm_size', + docker_settings.get( + 'shm_size', + '')) pass_user_id = i.get( - 'docker_pass_user_id', - docker_settings.get( - 'pass_user_id', - '')) + 'docker_pass_user_id', + docker_settings.get( + 'pass_user_id', + '')) pass_user_group = i.get( - 'docker_pass_user_group', - docker_settings.get( - 'pass_user_group', - '')) + 'docker_pass_user_group', + docker_settings.get( + 'pass_user_group', + '')) extra_run_args = i.get( - 'docker_extra_run_args', - docker_settings.get( - 'extra_run_args', - '')) + 'docker_extra_run_args', + docker_settings.get( + 'extra_run_args', + '')) if detached == '': detached = docker_settings.get('detached', '') - if str(docker_skip_run_cmd).lower() in ['true', '1','yes']: + if str(docker_skip_run_cmd).lower() in ['true', '1', 'yes']: interactive = 'yes' elif interactive == '': interactive = docker_settings.get('interactive', '') @@ -2259,24 +2382,24 @@ def docker(i): # else: # run_cmd = "" - - r = regenerate_script_cmd({'script_uid': script_uid, 'script_alias': script_alias, 'tags': tags, 'run_cmd': i_run_cmd, 'docker_settings': docker_settings, - 'docker_run_cmd_prefix': i.get('docker_run_cmd_prefix','')}) - if r['return']>0: + 'docker_run_cmd_prefix': i.get('docker_run_cmd_prefix', '')}) + if r['return'] > 0: return r - run_cmd = r['run_cmd_string'] + ' ' + container_env_string + ' --docker_run_deps ' + run_cmd = r['run_cmd_string'] + ' ' + \ + container_env_string + ' --docker_run_deps ' env['CM_RUN_STATE_DOCKER'] = True - if docker_settings.get('mount_current_dir','') =='yes': - run_cmd = 'cd ' +current_path_target +' && ' +run_cmd + if docker_settings.get('mount_current_dir', '') == 'yes': + run_cmd = 'cd ' + current_path_target + ' && ' + run_cmd - final_run_cmd = run_cmd if docker_skip_run_cmd not in ['yes', True, 'True' ] else 'cm version' + final_run_cmd = run_cmd if docker_skip_run_cmd not in [ + 'yes', True, 'True'] else 'cm version' print('') print('CM command line regenerated to be used inside Docker:') @@ -2354,22 +2477,25 @@ def docker(i): if r['return'] > 0: return r - return {'return': 0} ############################################################ + def check_gh_token(i, docker_settings, quiet): gh_token = i.get('docker_gh_token', '') if docker_settings.get('gh_token_required', False) and gh_token == '': - rx = {'return': 1, 'error':'GH token is required but not provided. Use --docker_gh_token to set it'} + rx = { + 'return': 1, + 'error': 'GH token is required but not provided. Use --docker_gh_token to set it'} if quiet: return rx print('') - gh_token = input('Enter GitHub token to access private CM repositories required for this CM script: ') + gh_token = input( + 'Enter GitHub token to access private CM repositories required for this CM script: ') if gh_token == '': return rx diff --git a/automation/script/template-ae-python/customize.py b/automation/script/template-ae-python/customize.py index d12f9b3e1d..273999d460 100644 --- a/automation/script/template-ae-python/customize.py +++ b/automation/script/template-ae-python/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,10 +14,11 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/automation/script/template-ae-python/main.py b/automation/script/template-ae-python/main.py index d851f1450f..caa499bf08 100644 --- a/automation/script/template-ae-python/main.py +++ b/automation/script/template-ae-python/main.py @@ -2,9 +2,9 @@ if __name__ == "__main__": - print ('') - print ('Main script:') - print ('Experiment: {}'.format(os.environ.get('CM_EXPERIMENT',''))) - print ('') + print('') + print('Main script:') + print('Experiment: {}'.format(os.environ.get('CM_EXPERIMENT', ''))) + print('') exit(0) diff --git a/automation/script/template-python/main.py b/automation/script/template-python/main.py index 9ba7bb751d..e3302f36fa 100644 --- a/automation/script/template-python/main.py +++ b/automation/script/template-python/main.py @@ -2,9 +2,9 @@ if __name__ == "__main__": - print ('') - print ('Main script:') - print ('ENV CM_VAR1: {}'.format(os.environ.get('CM_VAR1',''))) - print ('') + print('') + print('Main script:') + print('ENV CM_VAR1: {}'.format(os.environ.get('CM_VAR1', ''))) + print('') exit(0) diff --git a/automation/script/template-pytorch/main.py b/automation/script/template-pytorch/main.py index 3e49da450f..217aed3b9d 100644 --- a/automation/script/template-pytorch/main.py +++ b/automation/script/template-pytorch/main.py @@ -4,12 +4,12 @@ if __name__ == "__main__": - print ('') - print ('Main script:') - print ('ENV CM_VAR1: {}'.format(os.environ.get('CM_VAR1',''))) - print ('ENV USE_CUDA: {}'.format(os.environ.get('USE_CUDA',''))) - print ('') - print ('PyTorch version: {}'.format(torch.__version__)) - print ('') + print('') + print('Main script:') + print('ENV CM_VAR1: {}'.format(os.environ.get('CM_VAR1', ''))) + print('ENV USE_CUDA: {}'.format(os.environ.get('USE_CUDA', ''))) + print('') + print('PyTorch version: {}'.format(torch.__version__)) + print('') exit(0) diff --git a/automation/script/template/customize.py b/automation/script/template/customize.py index d12f9b3e1d..273999d460 100644 --- a/automation/script/template/customize.py +++ b/automation/script/template/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,10 +14,11 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/debug.py b/debug.py index 9fd54803c3..db6003d731 100644 --- a/debug.py +++ b/debug.py @@ -5,5 +5,6 @@ print(sys.executable) -r = cmind.access('run script "print hello-world python" --debug_uid=f52670e5f3f345a2') +r = cmind.access( + 'run script "print hello-world python" --debug_uid=f52670e5f3f345a2') print(r) diff --git a/script/activate-python-venv/customize.py b/script/activate-python-venv/customize.py index 938a016a05..5858212f93 100644 --- a/script/activate-python-venv/customize.py +++ b/script/activate-python-venv/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,17 +14,18 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - name = env.get('CM_NAME','') + name = env.get('CM_NAME', '') if name != '': name = name.strip().lower() - r = automation.update_deps({'deps':meta['prehook_deps'], - 'update_deps':{ - 'python-venv':{ - 'name':name - } - } - }) - if r['return']>0: return r - - return {'return':0} + r = automation.update_deps({'deps': meta['prehook_deps'], + 'update_deps': { + 'python-venv': { + 'name': name + } + } + }) + if r['return'] > 0: + return r + + return {'return': 0} diff --git a/script/add-custom-nvidia-system/customize.py b/script/add-custom-nvidia-system/customize.py index e9573338b1..dbab0597ea 100644 --- a/script/add-custom-nvidia-system/customize.py +++ b/script/add-custom-nvidia-system/customize.py @@ -2,16 +2,18 @@ import os import shutil + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] - return {'return':0} + return {'return': 0} + def postprocess(i): @@ -19,4 +21,4 @@ def postprocess(i): env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_MLPERF_INFERENCE_NVIDIA_CODE_PATH'] - return {'return':0} + return {'return': 0} diff --git a/script/app-image-corner-detection/customize.py b/script/app-image-corner-detection/customize.py index 88d65d1534..de1e344eeb 100644 --- a/script/app-image-corner-detection/customize.py +++ b/script/app-image-corner-detection/customize.py @@ -1,14 +1,15 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] env = i['env'] - script_path=i['run_script_input']['path'] + script_path = i['run_script_input']['path'] env["CM_SOURCE_FOLDER_PATH"] = script_path - env['CM_C_SOURCE_FILES']="susan.c" + env['CM_C_SOURCE_FILES'] = "susan.c" if 'CM_INPUT' not in env: env['CM_INPUT'] = os.path.join(script_path, 'data.pgm') @@ -18,24 +19,25 @@ def preprocess(i): if 'CM_RUN_DIR' not in env: output_path = os.path.join(script_path, "output") - if output_path!='' and not os.path.isdir(output_path): + if output_path != '' and not os.path.isdir(output_path): os.makedirs(output_path) env['CM_RUN_DIR'] = output_path - env['CM_RUN_SUFFIX']= env['CM_INPUT'] + ' ' + env['CM_OUTPUT'] + ' -c' + env['CM_RUN_SUFFIX'] = env['CM_INPUT'] + ' ' + env['CM_OUTPUT'] + ' -c' if os_info['platform'] == 'windows': - env['CM_BIN_NAME']='image-corner.exe' + env['CM_BIN_NAME'] = 'image-corner.exe' else: - env['CM_BIN_NAME']='image-corner' + env['CM_BIN_NAME'] = 'image-corner' env['+ LDCFLAGS'] = ["-lm"] - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] print(env['CM_OUTPUT'] + " generated in " + env['CM_RUN_DIR']) - return {'return':0} + return {'return': 0} diff --git a/script/app-loadgen-generic-python/src/backend_onnxruntime.py b/script/app-loadgen-generic-python/src/backend_onnxruntime.py index e95e467b9f..371f44ffbe 100644 --- a/script/app-loadgen-generic-python/src/backend_onnxruntime.py +++ b/script/app-loadgen-generic-python/src/backend_onnxruntime.py @@ -45,7 +45,7 @@ def __init__( inter_op_threads=0, model_code='', # Not used here model_cfg={}, # Not used here - model_sample_pickle='' # Not used here + model_sample_pickle='' # Not used here ): self.model_path = model_path self.execution_provider = execution_provider @@ -58,11 +58,12 @@ def __init__( self.session_options.inter_op_num_threads = inter_op_threads def create(self) -> Model: - print ('Loading model: {}'.format(self.model_path)) + print('Loading model: {}'.format(self.model_path)) # model = onnx.load(self.model_path) session_eps = [self.execution_provider] session = ort.InferenceSession( -# model.SerializeToString(), self.session_options, providers=session_eps + # model.SerializeToString(), self.session_options, + # providers=session_eps self.model_path, self.session_options, providers=session_eps ) return XModel(session) @@ -72,12 +73,14 @@ class XModelInputSampler(ModelInputSampler): def __init__(self, model_factory: XModelFactory): model = model_factory.create() input_defs = model.session.get_inputs() - self.inputs: typing.Dict[str, typing.Tuple[np.dtype, typing.List[int]]] = dict() + self.inputs: typing.Dict[str, + typing.Tuple[np.dtype, + typing.List[int]]] = dict() for input in input_defs: input_name = input.name input_type = ONNX_TO_NP_TYPE_MAP[input.type] input_dim = [ - 1 if (x is None or (type(x) is str)) else x for x in input.shape + 1 if (x is None or (isinstance(x, str))) else x for x in input.shape ] self.inputs[input_name] = (input_type, input_dim) diff --git a/script/app-loadgen-generic-python/src/loadgen/harness.py b/script/app-loadgen-generic-python/src/loadgen/harness.py index 69edd2ba95..a8fdd4e86b 100644 --- a/script/app-loadgen-generic-python/src/loadgen/harness.py +++ b/script/app-loadgen-generic-python/src/loadgen/harness.py @@ -58,7 +58,8 @@ def issue_query(self, query_samples): self._complete_query(result) # Called after the last call to issue queries in a series is made. - # Client can use this to flush any deferred queries rather than waiting for a timeout. + # Client can use this to flush any deferred queries rather than waiting + # for a timeout. def flush_queries(self): result = self.runner.flush_queries() logger.info(f"Queries flushed") diff --git a/script/app-loadgen-generic-python/src/loadgen/runners.py b/script/app-loadgen-generic-python/src/loadgen/runners.py index 1b78acba15..9c813a0278 100644 --- a/script/app-loadgen-generic-python/src/loadgen/runners.py +++ b/script/app-loadgen-generic-python/src/loadgen/runners.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) -######## Runner implementations +# Runner implementations class ModelRunnerInline(ModelRunner): @@ -172,7 +172,9 @@ def flush_queries(self) -> typing.Optional[QueryResult]: return result else: task_result = self.task.get() - result = {query_id: query_result for query_id, query_result in task_result} + result = { + query_id: query_result for query_id, + query_result in task_result} return result @staticmethod diff --git a/script/app-mlperf-inference-dummy/customize.py b/script/app-mlperf-inference-dummy/customize.py index 36f310babc..74ff72927e 100644 --- a/script/app-mlperf-inference-dummy/customize.py +++ b/script/app-mlperf-inference-dummy/customize.py @@ -2,32 +2,37 @@ import os import shutil + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] if env.get('CM_MLPERF_SKIP_RUN', '') == "yes": - return {'return':0} + return {'return': 0} if 'CM_MODEL' not in env: - return {'return': 1, 'error': 'Please select a variation specifying the model to run'} + return { + 'return': 1, 'error': 'Please select a variation specifying the model to run'} if 'CM_MLPERF_BACKEND' not in env: - return {'return': 1, 'error': 'Please select a variation specifying the backend'} + return {'return': 1, + 'error': 'Please select a variation specifying the backend'} if 'CM_MLPERF_DEVICE' not in env: - return {'return': 1, 'error': 'Please select a variation specifying the device to run on'} + return { + 'return': 1, 'error': 'Please select a variation specifying the device to run on'} r = get_run_cmd(env['CM_MODEL'], i) if r['return'] > 0: return r run_cmd = r['run_cmd'] - run_dir = r ['run_dir'] + run_dir = r['run_dir'] print(run_cmd) print(run_dir) - return {'return':1, 'error': 'Run command needs to be tested!'} + return {'return': 1, 'error': 'Run command needs to be tested!'} + def get_run_cmd(model, i): env = i['env'] @@ -49,12 +54,18 @@ def get_run_cmd(model, i): run_cmd = f"python3 -u main.py --scenario {scenario} --model-path {model_path} --api-server {api_server} --api-model-name gpt-j-cnn --mlperf-conf {mlperf_conf_path} {accuracy_string} --vllm --user-conf {user_conf_path} --dataset-path {dataset_path} --output-log-dir {outdir} --dtype float32 --device {device} " submitter = "CTuning" - run_dir = os.path.join(env['CM_MLPERF_INFERENCE_IMPLEMENTATION_REPO'], "open", submitter, "code", "gptj-99") + run_dir = os.path.join( + env['CM_MLPERF_INFERENCE_IMPLEMENTATION_REPO'], + "open", + submitter, + "code", + "gptj-99") return {'return': 0, 'run_cmd': run_cmd, 'run_dir': run_dir} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/app-mlperf-inference-mlcommons-python/nvidia/retinanet.py b/script/app-mlperf-inference-mlcommons-python/nvidia/retinanet.py index 705f1e3539..090d1b072a 100644 --- a/script/app-mlperf-inference-mlcommons-python/nvidia/retinanet.py +++ b/script/app-mlperf-inference-mlcommons-python/nvidia/retinanet.py @@ -30,7 +30,7 @@ from typing import Dict, Tuple, List, Optional from code.common.fix_sys_path import ScopedRestrictedImport -#with ScopedRestrictedImport(): +# with ScopedRestrictedImport(): import numpy as np import torch # Retinanet model source requires GPU installation of PyTorch 1.10 from torchvision.transforms import functional as F @@ -45,16 +45,27 @@ from code.common.runner import EngineRunner, get_input_format from code.common.systems.system_list import SystemClassifications from code.plugin import load_trt_plugin -RetinanetEntropyCalibrator = import_module("code.retinanet.tensorrt.calibrator").RetinanetEntropyCalibrator +RetinanetEntropyCalibrator = import_module( + "code.retinanet.tensorrt.calibrator").RetinanetEntropyCalibrator G_RETINANET_NUM_CLASSES = 264 G_RETINANET_IMG_SIZE = (800, 800) G_RETINANET_INPUT_SHAPE = (3, 800, 800) G_OPENIMAGE_CALSET_PATH = "build/data/open-images-v6-mlperf/calibration/train/data" G_OPENIMAGE_CALMAP_PATH = "data_maps/open-images-v6-mlperf/cal_map.txt" -G_OPENIMAGE_VALSET_PATH = os.path.join(os.environ.get("CM_DATASET_PATH", "build/data/open-images-v6-mlperf"), "validation", "data") +G_OPENIMAGE_VALSET_PATH = os.path.join( + os.environ.get( + "CM_DATASET_PATH", + "build/data/open-images-v6-mlperf"), + "validation", + "data") G_OPENIMAGE_VALMAP_PATH = "data_maps/open-images-v6-mlperf/val_map.txt" -G_OPENIMAGE_ANNO_PATH = os.path.join(os.environ.get("CM_DATASET_PATH","build/data/open-images-v6-mlperf"), "annotations", "openimages-mlperf.json") +G_OPENIMAGE_ANNO_PATH = os.path.join( + os.environ.get( + "CM_DATASET_PATH", + "build/data/open-images-v6-mlperf"), + "annotations", + "openimages-mlperf.json") G_OPENIMAGE_PREPROCESSED_INT8_PATH = "build/preprocessed_data/open-images-v6-mlperf/validation/Retinanet/int8_linear" # Using EfficientNMS now G_RETINANET_CALIBRATION_CACHE_PATH = "code/retinanet/tensorrt/calibrator.cache" @@ -91,7 +102,8 @@ class FirstLayerConvActPoolTacticSelector(trt.IAlgorithmSelector): def select_algorithms(self, ctx, choices): if "Conv_0 + 1783 + Mul_1 + 1785 + Add_2 + Relu_3 + MaxPool_4" in ctx.name: # Apply to the first layer # MLPINF-1833: Disabled CaskConvActPool for TRT 8.5.0.4 - # TRT 8.5.0.4 has a bug with CaskConvActPool which has been fixed since 8.5.0.5 + # TRT 8.5.0.4 has a bug with CaskConvActPool which has been fixed + # since 8.5.0.5 forbidden_set = { -3689373275198309793, # 0xccccb68da7fc3a5f -4219016963003938541, # 0xc5730a6ceacd8913 @@ -116,7 +128,8 @@ def select_algorithms(self, ctx, choices): -7700711094551245800, # 0xf126325c0aa4aa02 -1070112490556970494, # 0x97d50e90c139753e } - filtered_idxs = [idx for idx, choice in enumerate(choices) if choice.algorithm_variant.tactic not in forbidden_set] + filtered_idxs = [idx for idx, choice in enumerate( + choices) if choice.algorithm_variant.tactic not in forbidden_set] to_ret = filtered_idxs else: # By default, say that all tactics are acceptable: @@ -149,7 +162,8 @@ def __init__(self, engine_file, batch_size, precision, onnx_path, self.dla_core = None # Initiate the plugin and logger - self.logger = TRT_LOGGER # Use the global singleton, which is required by TRT. + # Use the global singleton, which is required by TRT. + self.logger = TRT_LOGGER self.logger.min_severity = trt.Logger.VERBOSE if self.verbose else trt.Logger.INFO load_trt_plugin("retinanet") trt.init_libnvinfer_plugins(self.logger, "") @@ -159,7 +173,8 @@ def __init__(self, engine_file, batch_size, precision, onnx_path, self.create_trt_engine() else: if not os.path.exists(engine_file): - raise RuntimeError(f"Cannot find engine file {engine_file}. Please supply the onnx file or engine file.") + raise RuntimeError( + f"Cannot find engine file {engine_file}. Please supply the onnx file or engine file.") self.runner = EngineRunner(self.engine_file, verbose=verbose) @@ -170,11 +185,15 @@ def __init__(self, engine_file, batch_size, precision, onnx_path, def apply_flag(self, flag): """Apply a TRT builder flag.""" - self.builder_config.flags = (self.builder_config.flags) | (1 << int(flag)) + self.builder_config.flags = ( + self.builder_config.flags) | ( + 1 << int(flag)) def clear_flag(self, flag): """Clear a TRT builder flag.""" - self.builder_config.flags = (self.builder_config.flags) & ~(1 << int(flag)) + self.builder_config.flags = ( + self.builder_config.flags) & ~( + 1 << int(flag)) # Helper function to build a TRT engine from ONNX file def create_trt_engine(self): @@ -195,7 +214,9 @@ def create_trt_engine(self): # Calibrator for int8 preprocessed_data_dir = "build/preprocessed_data" - calib_image_dir = os.path.join(preprocessed_data_dir, "open-images-v6-mlperf/calibration/Retinanet/fp32") + calib_image_dir = os.path.join( + preprocessed_data_dir, + "open-images-v6-mlperf/calibration/Retinanet/fp32") self.calibrator = RetinanetEntropyCalibrator(data_dir=calib_image_dir, cache_file=self.cache_file, batch_size=10, max_batches=50, force_calibration=False, calib_data_map=G_OPENIMAGE_CALMAP_PATH) @@ -208,13 +229,15 @@ def create_trt_engine(self): else: raise Exception(f"{self.precision} not supported yet.") - self.network = self.builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)) + self.network = self.builder.create_network( + 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)) model = onnx.load(self.onnx_path) parser = trt.OnnxParser(self.network, self.logger) success = parser.parse(onnx._serialize(model)) if not success: err_desc = parser.get_error(0).desc() - raise RuntimeError(f"Retinanet onnx model processing failed! Error: {err_desc}") + raise RuntimeError( + f"Retinanet onnx model processing failed! Error: {err_desc}") # Set the network input type if self.precision == "int8": @@ -236,7 +259,8 @@ def create_trt_engine(self): min_shape[0] = 1 max_shape = trt.Dims(input_shape) max_shape[0] = self.batch_size - profile.set_shape(input_name, min_shape, max_shape, max_shape) + profile.set_shape( + input_name, min_shape, max_shape, max_shape) if not profile: raise RuntimeError("Invalid optimization profile!") self.builder_config.add_optimization_profile(profile) @@ -250,7 +274,8 @@ def create_trt_engine(self): engine = self.builder.build_engine(self.network, self.builder_config) engine_inspector = engine.create_engine_inspector() - layer_info = engine_inspector.get_engine_information(trt.LayerInformationFormat.ONELINE) + layer_info = engine_inspector.get_engine_information( + trt.LayerInformationFormat.ONELINE) logging.info("========= TensorRT Engine Layer Information =========") logging.info(layer_info) @@ -264,14 +289,16 @@ def run_openimage(self, num_samples=8): image_ids = cocoGt.getImgIds() cat_ids = cocoGt.getCatIds() num_images = min(num_samples, len(image_ids)) - print(f"Total number of images: {len(image_ids)}, number of categories: {len(cat_ids)}, running num_images: {num_images}") + print( + f"Total number of images: {len(image_ids)}, number of categories: {len(cat_ids)}, running num_images: {num_images}") detections = [] batch_idx = 0 for image_idx in range(0, num_images, self.batch_size): # Print Progress if batch_idx % 20 == 0: - print(f"Processing batch: {batch_idx} image: {image_idx}/{num_images}") + print( + f"Processing batch: {batch_idx} image: {image_idx}/{num_images}") end_idx = min(image_idx + self.batch_size, num_images) imgs = [] @@ -279,14 +306,24 @@ def run_openimage(self, num_samples=8): for idx in range(image_idx, end_idx): image_id = image_ids[idx] if self.precision == "fp32": - # Load the image using pytorch routine, but perform extra resize+normalize steps - img = load_img_pytorch(os.path.join(self.image_dir, cocoGt.imgs[image_id]["file_name"]), do_transform=True).numpy() + # Load the image using pytorch routine, but perform extra + # resize+normalize steps + img = load_img_pytorch( + os.path.join( + self.image_dir, + cocoGt.imgs[image_id]["file_name"]), + do_transform=True).numpy() elif self.precision == "int8": - img = np.load(os.path.join(G_OPENIMAGE_PREPROCESSED_INT8_PATH, cocoGt.imgs[image_id]["file_name"] + '.npy')) + img = np.load( + os.path.join( + G_OPENIMAGE_PREPROCESSED_INT8_PATH, + cocoGt.imgs[image_id]["file_name"] + + '.npy')) else: raise Exception(f"Unsupported precision {self.precision}") imgs.append(img) - img_original_sizes.append([cocoGt.imgs[image_id]["height"], cocoGt.imgs[image_id]["width"]]) + img_original_sizes.append( + [cocoGt.imgs[image_id]["height"], cocoGt.imgs[image_id]["width"]]) if self.precision == "fp32": imgs = np.ascontiguousarray(np.stack(imgs), dtype=np.float32) @@ -298,7 +335,8 @@ def run_openimage(self, num_samples=8): if self.verbose: duration = time.time() - start_time - logging.info(f"Batch {batch_idx} >>> Inference time: {duration}") + logging.info( + f"Batch {batch_idx} >>> Inference time: {duration}") # Concatted outputs is in the shape of [BS, 7001] # image_ids (duplicate of score for loadgen): [BS, 1000, 1] @@ -318,7 +356,14 @@ def run_openimage(self, num_samples=8): for prediction_idx in range(0, keep_count): # Each detection is in the order of [dummy_image_idx, xmin, ymin, xmax, ymax, score, label] # This is pre-callback (otherwise x and y are swapped). - single_detection = concat_output[idx * 7001 + prediction_idx * 7: idx * 7001 + prediction_idx * 7 + 7] + single_detection = concat_output[idx * + 7001 + + prediction_idx * + 7: idx * + 7001 + + prediction_idx * + 7 + + 7] loc = single_detection[1:5] label = single_detection[6] score = single_detection[5] @@ -371,8 +416,10 @@ class PytorchTester: To run this tester, you would need to clone the repo, and mount it to the container. """ - def __init__(self, pyt_ckpt_path, training_repo_path, batch_size=8, output_file="build/retinanet_pytorch.out"): - ssd_model_path = os.path.join(training_repo_path, "single_stage_detector", "ssd") + def __init__(self, pyt_ckpt_path, training_repo_path, + batch_size=8, output_file="build/retinanet_pytorch.out"): + ssd_model_path = os.path.join( + training_repo_path, "single_stage_detector", "ssd") with ScopedRestrictedImport([ssd_model_path] + sys.path): from model.retinanet import retinanet_from_backbone pyt_model = retinanet_from_backbone( @@ -409,7 +456,8 @@ def run_openimage(self, num_samples=8): image_ids = cocoGt.getImgIds() cat_ids = cocoGt.getCatIds() num_images = min(num_samples, len(image_ids)) - print(f"Total number of images: {len(image_ids)}, number of categories: {len(cat_ids)}, running num_images: {num_images}") + print( + f"Total number of images: {len(image_ids)}, number of categories: {len(cat_ids)}, running num_images: {num_images}") coco_detections = [] for image_idx in range(0, num_images, self.batch_size): @@ -418,7 +466,8 @@ def run_openimage(self, num_samples=8): imgs = [] for idx in range(image_idx, end_idx): image_id = image_ids[idx] - image_path = os.path.join(self.image_dir, cocoGt.imgs[image_id]["file_name"]) + image_path = os.path.join( + self.image_dir, cocoGt.imgs[image_id]["file_name"]) img = load_img_pytorch(image_path).to(self.device) imgs.append(img) # print(cocoGt.imgs[image_id]["height"], cocoGt.imgs[image_id]["width"]) @@ -426,7 +475,11 @@ def run_openimage(self, num_samples=8): img = [] for idx in range(image_idx, end_idx): image_id = image_ids[idx] - tensor = load_img_pytorch(os.path.join(self.image_dir, cocoGt.imgs[image_id]["file_name"]), do_transform=True).numpy() + tensor = load_img_pytorch( + os.path.join( + self.image_dir, + cocoGt.imgs[image_id]["file_name"]), + do_transform=True).numpy() print(tensor.shape) img.append(tensor) img = np.ascontiguousarray(np.stack(img), dtype=np.float32) @@ -445,7 +498,8 @@ def run_openimage(self, num_samples=8): # Convert from lrtb to [xmin, ymin, w, h] for cocoeval box_pred = boxes[pred_idx][:] xmin, ymin, xmax, ymax = box_pred - box_pred = np.array([xmin, ymin, xmax - xmin, ymax - ymin], dtype=np.float32) + box_pred = np.array( + [xmin, ymin, xmax - xmin, ymax - ymin], dtype=np.float32) score_pred = float(scores[pred_idx]) label_pred = int(labels[pred_idx]) coco_detection = { @@ -526,24 +580,40 @@ def main(): # Pytorch Tester if args.pytorch: # TODO: Check existence of training repo. - logging.info(f"Running Accuracy test for Pytorch reference implementation.") - if args.training_repo_path is None or not os.path.exists(args.training_repo_path): - raise RuntimeError("Please pull mlcommon training repo from https://github.com/mlcommons/training, and specify with --training_repo_path") - pt_tester = PytorchTester(args.pyt_ckpt_path, args.training_repo_path, args.batch_size) + logging.info( + f"Running Accuracy test for Pytorch reference implementation.") + if args.training_repo_path is None or not os.path.exists( + args.training_repo_path): + raise RuntimeError( + "Please pull mlcommon training repo from https://github.com/mlcommons/training, and specify with --training_repo_path") + pt_tester = PytorchTester( + args.pyt_ckpt_path, + args.training_repo_path, + args.batch_size) pt_acc = pt_tester.run_openimage(args.num_samples) - logging.info(f"Pytorch mAP Score: {pt_acc}, Reference: 0.375, % of ref: {pt_acc / 0.375}") + logging.info( + f"Pytorch mAP Score: {pt_acc}, Reference: 0.375, % of ref: {pt_acc / 0.375}") else: # TRT Tester - logging.info(f"Running accuracy test for retinanet using {args.engine_file} ...") - tester = TRTTester(args.engine_file, args.batch_size, args.trt_precision, args.onnx_path, args.skip_engine_build, args.verbose) + logging.info( + f"Running accuracy test for retinanet using {args.engine_file} ...") + tester = TRTTester( + args.engine_file, + args.batch_size, + args.trt_precision, + args.onnx_path, + args.skip_engine_build, + args.verbose) # acc = tester.run_openimage(args.num_samples) acc = tester.run_openimage(args.num_samples) - logging.info(f"mAP Score: {acc}, Reference: 0.375, % of ref: {acc / 0.375}") + logging.info( + f"mAP Score: {acc}, Reference: 0.375, % of ref: {acc / 0.375}") # To run the TRT tester: # python3 -m code.retinanet.tensorrt.infer --engine_file /tmp/retina.b8.int8.engine --num_samples=1200 --batch_size=8 --trt_precision int8 # To run the pytorch tester: - # python3 -m code.retinanet.tensorrt.infer --pytorch --num_samples=1200 --batch_size=8 + # python3 -m code.retinanet.tensorrt.infer --pytorch --num_samples=1200 + # --batch_size=8 if __name__ == "__main__": diff --git a/script/app-stable-diffusion-onnx-py/process.py b/script/app-stable-diffusion-onnx-py/process.py index 0f759089b7..86bbd3c3b6 100644 --- a/script/app-stable-diffusion-onnx-py/process.py +++ b/script/app-stable-diffusion-onnx-py/process.py @@ -4,31 +4,33 @@ from optimum.onnxruntime import ORTStableDiffusionPipeline -output = os.environ.get('CM_APP_STABLE_DIFFUSION_ONNX_PY_OUTPUT','') +output = os.environ.get('CM_APP_STABLE_DIFFUSION_ONNX_PY_OUTPUT', '') f = os.path.join(output, 'output.png') if os.path.isfile(f): os.remove(f) -cm_model_path = os.environ.get('CM_ML_MODEL_PATH','') +cm_model_path = os.environ.get('CM_ML_MODEL_PATH', '') if cm_model_path == '': - print ('Error: CM_ML_MODEL_PATH env is not defined') + print('Error: CM_ML_MODEL_PATH env is not defined') exit(1) -device = os.environ.get('CM_DEVICE','') +device = os.environ.get('CM_DEVICE', '') -pipeline = ORTStableDiffusionPipeline.from_pretrained(cm_model_path, local_files_only=True).to(device) +pipeline = ORTStableDiffusionPipeline.from_pretrained( + cm_model_path, local_files_only=True).to(device) -text = os.environ.get('CM_APP_STABLE_DIFFUSION_ONNX_PY_TEXT','') -if text == '': text = "a photo of an astronaut riding a horse on mars" +text = os.environ.get('CM_APP_STABLE_DIFFUSION_ONNX_PY_TEXT', '') +if text == '': + text = "a photo of an astronaut riding a horse on mars" -print ('') -print ('Generating imaged based on "{}"'.format(text)) +print('') +print('Generating imaged based on "{}"'.format(text)) image = pipeline(text).images[0] image.save(f) -print ('Image recorded to "{}"'.format(f)) +print('Image recorded to "{}"'.format(f)) diff --git a/script/authenticate-github-cli/customize.py b/script/authenticate-github-cli/customize.py index a873791f43..f4adae9931 100644 --- a/script/authenticate-github-cli/customize.py +++ b/script/authenticate-github-cli/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -23,10 +24,11 @@ def preprocess(i): env['CM_RUN_CMD'] = cmd quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/build-mlperf-inference-server-nvidia/customize.py b/script/build-mlperf-inference-server-nvidia/customize.py index 47338aed90..8d477fcc74 100644 --- a/script/build-mlperf-inference-server-nvidia/customize.py +++ b/script/build-mlperf-inference-server-nvidia/customize.py @@ -2,29 +2,34 @@ import os import shutil + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] if '+LIBRARY_PATH' not in env: env['+LIBRARY_PATH'] = [] if 'CM_TENSORRT_INSTALL_PATH' in env: - env['+LIBRARY_PATH'].append(os.path.join(env['CM_TENSORRT_INSTALL_PATH'], "lib")) + env['+LIBRARY_PATH'].append(os.path.join( + env['CM_TENSORRT_INSTALL_PATH'], "lib")) - cxxflags = [ "-Wno-error=switch", "-DDALI_1_15=1", "-Wno-error=maybe-uninitialized" ] + cxxflags = [ + "-Wno-error=switch", + "-DDALI_1_15=1", + "-Wno-error=maybe-uninitialized"] if env.get('CM_GCC_VERSION', '') != '': gcc_major_version = env['CM_GCC_VERSION'].split(".")[0] if int(gcc_major_version) > 10: - if env.get('CM_MLPERF_INFERENCE_VERSION','') != "4.1": + if env.get('CM_MLPERF_INFERENCE_VERSION', '') != "4.1": cxxflags.append("-Wno-error=range-loop-construct") - if env.get('CM_MLPERF_DEVICE','') == "inferentia": + if env.get('CM_MLPERF_DEVICE', '') == "inferentia": env['USE_INFERENTIA'] = "1" env['USE_NIGHTLY'] = "0" env['CM_MAKE_BUILD_COMMAND'] = "build" @@ -33,10 +38,11 @@ def preprocess(i): env['+ CXXFLAGS'] = [] env['+ CXXFLAGS'] += cxxflags - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/clean-nvidia-mlperf-inference-scratch-space/customize.py b/script/clean-nvidia-mlperf-inference-scratch-space/customize.py index 977a9993bb..11d224c59c 100644 --- a/script/clean-nvidia-mlperf-inference-scratch-space/customize.py +++ b/script/clean-nvidia-mlperf-inference-scratch-space/customize.py @@ -2,6 +2,7 @@ import os import cmind as cm + def preprocess(i): os_info = i['os_info'] @@ -21,32 +22,34 @@ def preprocess(i): if env.get('CM_MODEL', '') == 'sdxl': if env.get('CM_CLEAN_ARTIFACT_NAME', '') == 'downloaded_data': clean_cmd = f"""rm -rf {os.path.join(env['CM_NVIDIA_MLPERF_SCRATCH_PATH'], "data", "coco", "SDXL")} """ - cache_rm_tags = "nvidia-harness,_preprocess_data,_sdxl" + cache_rm_tags = "nvidia-harness,_preprocess_data,_sdxl" if env.get('CM_CLEAN_ARTIFACT_NAME', '') == 'preprocessed_data': clean_cmd = f"""rm -rf {os.path.join(env['CM_NVIDIA_MLPERF_SCRATCH_PATH'], "preprocessed_data", "coco2014-tokenized-sdxl")} """ - cache_rm_tags = "nvidia-harness,_preprocess_data,_sdxl" + cache_rm_tags = "nvidia-harness,_preprocess_data,_sdxl" if env.get('CM_CLEAN_ARTIFACT_NAME', '') == 'downloaded_model': clean_cmd = f"""rm -rf {os.path.join(env['CM_NVIDIA_MLPERF_SCRATCH_PATH'], "models", "SDXL")} """ - cache_rm_tags = "nvidia-harness,_download_model,_sdxl" + cache_rm_tags = "nvidia-harness,_download_model,_sdxl" cache_rm_tags = cache_rm_tags + extra_cache_rm_tags if cache_rm_tags: - r = cm.access({'action': 'rm', 'automation': 'cache', 'tags': cache_rm_tags, 'f': True}) + r = cm.access({'action': 'rm', 'automation': 'cache', + 'tags': cache_rm_tags, 'f': True}) print(r) - if r['return'] != 0 and r['return'] != 16: ## ignore missing ones + if r['return'] != 0 and r['return'] != 16: # ignore missing ones return r - if r['return'] == 0: # cache entry found + if r['return'] == 0: # cache entry found if clean_cmd != '': env['CM_RUN_CMD'] = clean_cmd else: if clean_cmd != '': env['CM_RUN_CMD'] = clean_cmd - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/compile-program/customize.py b/script/compile-program/customize.py index 73a3eeb82b..c5e2adabb8 100644 --- a/script/compile-program/customize.py +++ b/script/compile-program/customize.py @@ -1,29 +1,38 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] env = i['env'] CPPFLAGS = env.get('+ CPPFLAGS', []) env['CM_C_COMPILER_FLAGS'] = " ".join(env.get('+ CFLAGS', []) + CPPFLAGS) - env['CM_CXX_COMPILER_FLAGS'] = " ".join(env.get('+ CXXFLAGS', []) + CPPFLAGS) + env['CM_CXX_COMPILER_FLAGS'] = " ".join( + env.get('+ CXXFLAGS', []) + CPPFLAGS) env['CM_F_COMPILER_FLAGS'] = " ".join(env.get('+ FFLAGS', [])) - CPATH = env.get('+CPATH', [ ]) - env['CM_C_INCLUDE_PATH'] = " -I".join([" "] + env.get('+C_INCLUDE_PATH', []) + CPATH) - env['CM_CPLUS_INCLUDE_PATH'] = " -I".join([" "] + env.get('+CPLUS_INCLUDE_PATH', []) + CPATH) - env['CM_F_INCLUDE_PATH'] = " -I".join([" "] + env.get('+F_INCLUDE_PATH', []) + CPATH) + CPATH = env.get('+CPATH', []) + env['CM_C_INCLUDE_PATH'] = " -I".join([" "] + + env.get('+C_INCLUDE_PATH', []) + + CPATH) + env['CM_CPLUS_INCLUDE_PATH'] = " -I".join( + [" "] + env.get('+CPLUS_INCLUDE_PATH', []) + CPATH) + env['CM_F_INCLUDE_PATH'] = " -I".join([" "] + + env.get('+F_INCLUDE_PATH', []) + + CPATH) # If windows, need to extend it more ... - if os_info['platform'] == 'windows' and env.get('CM_COMPILER_FAMILY','')!='LLVM': - print ("WARNING: compile-program script should be extended to support flags for non-LLVM compilers on Windows") - return {'return':0} + if os_info['platform'] == 'windows' and env.get( + 'CM_COMPILER_FAMILY', '') != 'LLVM': + print("WARNING: compile-program script should be extended to support flags for non-LLVM compilers on Windows") + return {'return': 0} LDFLAGS = env.get('+ LDFLAGS', []) env['CM_C_LINKER_FLAGS'] = " ".join(env.get('+ LDCFLAGS', []) + LDFLAGS) - env['CM_CXX_LINKER_FLAGS'] = " ".join(env.get('+ LDCXXFLAGS', []) + LDFLAGS) + env['CM_CXX_LINKER_FLAGS'] = " ".join( + env.get('+ LDCXXFLAGS', []) + LDFLAGS) env['CM_F_LINKER_FLAGS'] = " ".join(env.get('+ LDFFLAGS', []) + LDFLAGS) if env.get('CM_LINKER_LANG', 'C') == "C": @@ -44,11 +53,14 @@ def preprocess(i): env['CM_LINKER_COMPILE_FLAGS'] = env['CM_F_COMPILER_FLAGS'] env['CM_LINKER_FLAGS'] = env['CM_F_LINKER_FLAGS'] - env['CM_LD_LIBRARY_PATH'] = " -L".join([" " ] + env.get('+LD_LIBRARY_PATH', [])) - env['CM_SOURCE_FOLDER_PATH'] = env['CM_SOURCE_FOLDER_PATH'] if 'CM_SOURCE_FOLDER_PATH' in env else env['CM_TMP_CURRENT_SCRIPT_PATH'] if 'CM_TMP_CURRENT_SCRIPT_PATH' in env else '' + env['CM_LD_LIBRARY_PATH'] = " -L".join([" "] + + env.get('+LD_LIBRARY_PATH', [])) + env['CM_SOURCE_FOLDER_PATH'] = env['CM_SOURCE_FOLDER_PATH'] if 'CM_SOURCE_FOLDER_PATH' in env else env[ + 'CM_TMP_CURRENT_SCRIPT_PATH'] if 'CM_TMP_CURRENT_SCRIPT_PATH' in env else '' + + return {'return': 0} - return {'return':0} def postprocess(i): - return {'return':0} + return {'return': 0} diff --git a/script/copy-to-clipboard/code.py b/script/copy-to-clipboard/code.py index 082813e9a0..0a1aa014a0 100644 --- a/script/copy-to-clipboard/code.py +++ b/script/copy-to-clipboard/code.py @@ -3,7 +3,9 @@ text = os.environ.get('CM_COPY_TO_CLIPBOARD_TEXT', '') -add_quotes = os.environ.get('CM_COPY_TO_CLIPBOARD_TEXT_ADD_QUOTES', '') in [True,'True','yes'] +add_quotes = os.environ.get( + 'CM_COPY_TO_CLIPBOARD_TEXT_ADD_QUOTES', '') in [ + True, 'True', 'yes'] if add_quotes: text = '"' + text + '"' diff --git a/script/create-conda-env/customize.py b/script/create-conda-env/customize.py index 3d4b17e7fd..c9fd8083bf 100644 --- a/script/create-conda-env/customize.py +++ b/script/create-conda-env/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -12,9 +13,10 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] if env.get('CM_CONDA_ENV_NAME', '') == '': - return {'return':1, 'error': 'Please use "_name." variation'} + return {'return': 1, 'error': 'Please use "_name." variation'} + + return {'return': 0} - return {'return':0} def postprocess(i): env = i['env'] @@ -25,7 +27,7 @@ def postprocess(i): env['CM_CONDA_BIN_PATH'] = os.path.join(conda_prefix, "bin") env['CM_CONDA_LIB_PATH'] = os.path.join(conda_prefix, "lib") - env['+PATH'] = [ env['CM_CONDA_BIN_PATH'] ] - env['+LD_LIBRARY_PATH'] = [ env['CM_CONDA_LIB_PATH'] ] + env['+PATH'] = [env['CM_CONDA_BIN_PATH']] + env['+LD_LIBRARY_PATH'] = [env['CM_CONDA_LIB_PATH']] - return {'return':0} + return {'return': 0} diff --git a/script/create-patch/customize.py b/script/create-patch/customize.py index 2990d29ff0..0ebd63b99f 100644 --- a/script/create-patch/customize.py +++ b/script/create-patch/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -15,39 +16,42 @@ def preprocess(i): new_dir = env.get('CM_CREATE_PATCH_NEW', '') if new_dir == '': - return {'return':1, 'error':'specify NEW directory using --new'} + return {'return': 1, 'error': 'specify NEW directory using --new'} if not os.path.isdir(new_dir): - return {'return':1, 'error':'NEW directory doesn\'t exist {}'.format(new_dir)} + return {'return': 1, + 'error': 'NEW directory doesn\'t exist {}'.format(new_dir)} old_dir = env.get('CM_CREATE_PATCH_OLD', '') if old_dir == '': - return {'return':1, 'error':'specify OLD directory using --old'} + return {'return': 1, 'error': 'specify OLD directory using --old'} if not os.path.isdir(old_dir): - return {'return':1, 'error':'OLD directory doesn\'t exist {}'.format(old_dir)} + return {'return': 1, + 'error': 'OLD directory doesn\'t exist {}'.format(old_dir)} exclude = env.get('CM_CREATE_PATCH_EXCLUDE', '').strip() x_exclude = '' - if exclude!='': + if exclude != '': for e in exclude.split(','): - x_exclude+=' --exclude={}'.format(e) + x_exclude += ' --exclude={}'.format(e) - cmd = 'diff -Naur {} {} {} > patch.patch'.format(x_exclude, old_dir, new_dir) + cmd = 'diff -Naur {} {} {} > patch.patch'.format( + x_exclude, old_dir, new_dir) if not quiet: - print ('') - print ('Running command:') - print ('') - print (cmd) - print ('') + print('') + print('Running command:') + print('') + print(cmd) + print('') os.system(cmd) + return {'return': 0} - return {'return':0} def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/detect-os/customize.py b/script/detect-os/customize.py index d59a511f6a..c5a0b17c1d 100644 --- a/script/detect-os/customize.py +++ b/script/detect-os/customize.py @@ -2,6 +2,7 @@ import os import subprocess + def preprocess(i): env = i['env'] @@ -17,7 +18,7 @@ def preprocess(i): # Update state (demo) # state['os_info'] = os_info - return {'return':0} + return {'return': 0} def postprocess(i): @@ -30,7 +31,8 @@ def postprocess(i): if os_info['platform'] != 'windows': if os_info['platform'] == 'linux': sys_cmd = "ld --verbose | grep SEARCH_DIR " - result = subprocess.check_output(sys_cmd, shell=True).decode("utf-8") + result = subprocess.check_output( + sys_cmd, shell=True).decode("utf-8") result = result.replace("SEARCH_DIR(\"=", "") result = result.replace("SEARCH_DIR(\"", "") result = result.replace("\")", "") @@ -39,14 +41,15 @@ def postprocess(i): dirs = result.split(';') lib_dir = [] for _dir in dirs: - if _dir != '' and _dir not in lib_dir: + if _dir != '' and _dir not in lib_dir: lib_dir.append(_dir) env['+CM_HOST_OS_DEFAULT_LIBRARY_PATH'] = lib_dir r = utils.load_txt(file_name='tmp-run.out', - check_if_exists = True, - split = True) - if r['return']>0: return r + check_if_exists=True, + split=True) + if r['return'] > 0: + return r s = r['list'] @@ -63,20 +66,20 @@ def postprocess(i): env['CM_HOST_SYSTEM_NAME'] = platform.node() if 'CM_HOST_OS_PACKAGE_MANAGER' not in env: - if env.get('CM_HOST_OS_FLAVOR','') == "ubuntu" or \ - "debian" in env.get('CM_HOST_OS_FLAVOR_LIKE','') or \ - env.get('CM_HOST_OS_FLAVOR','') == "debian": + if env.get('CM_HOST_OS_FLAVOR', '') == "ubuntu" or \ + "debian" in env.get('CM_HOST_OS_FLAVOR_LIKE', '') or \ + env.get('CM_HOST_OS_FLAVOR', '') == "debian": env['CM_HOST_OS_PACKAGE_MANAGER'] = "apt" - if env.get('CM_HOST_OS_FLAVOR','') == "rhel" or \ - "rhel" in env.get('CM_HOST_OS_FLAVOR_LIKE',''): + if env.get('CM_HOST_OS_FLAVOR', '') == "rhel" or \ + "rhel" in env.get('CM_HOST_OS_FLAVOR_LIKE', ''): env['CM_HOST_OS_PACKAGE_MANAGER'] = "dnf" - if env.get('CM_HOST_OS_FLAVOR','') == "amzn": + if env.get('CM_HOST_OS_FLAVOR', '') == "amzn": env['CM_HOST_OS_PACKAGE_MANAGER'] = "yum" - if env.get('CM_HOST_OS_FLAVOR_LIKE','') == "arch": + if env.get('CM_HOST_OS_FLAVOR_LIKE', '') == "arch": env['CM_HOST_OS_PACKAGE_MANAGER'] = "arch" - if env.get('CM_HOST_OS_FLAVOR','') == "macos": + if env.get('CM_HOST_OS_FLAVOR', '') == "macos": env['CM_HOST_OS_PACKAGE_MANAGER'] = "brew" - if env.get('CM_HOST_OS_FLAVOR','') == "sles": + if env.get('CM_HOST_OS_FLAVOR', '') == "sles": env['CM_HOST_OS_PACKAGE_MANAGER'] = "zypper" if env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "apt": env['CM_HOST_OS_PACKAGE_MANAGER_INSTALL_CMD'] = "DEBIAN_FRONTEND=noninteractive apt-get install -y" @@ -103,4 +106,4 @@ def postprocess(i): if os.path.exists("/.dockerenv"): env['CM_RUN_INSIDE_DOCKER'] = "yes" - return {'return':0} + return {'return': 0} diff --git a/script/download-and-extract/customize.py b/script/download-and-extract/customize.py index 87d65b35af..b3c570dbc6 100644 --- a/script/download-and-extract/customize.py +++ b/script/download-and-extract/customize.py @@ -2,6 +2,7 @@ import os import hashlib + def preprocess(i): os_info = i['os_info'] @@ -18,32 +19,37 @@ def preprocess(i): extra_cache_tags = i['input'].get('extra_cache_tags', '') r = automation.update_deps({ 'deps': meta['prehook_deps'] + meta['posthook_deps'], - 'update_deps':{ + 'update_deps': { 'download-script': { 'extra_cache_tags': extra_cache_tags, 'force_cache': True - }, - 'extract-script':{ + }, + 'extract-script': { 'extra_cache_tags': extra_cache_tags, 'force_cache': True - } } - }) - if r['return']>0: return r + } + }) + if r['return'] > 0: + return r if env.get('CM_DOWNLOAD_LOCAL_FILE_PATH'): filepath = env['CM_DOWNLOAD_LOCAL_FILE_PATH'] if not os.path.exists(filepath): - return {'return':1, 'error':'Local file {} doesn\'t exist'.format(filepath)} + return {'return': 1, + 'error': 'Local file {} doesn\'t exist'.format(filepath)} - env['CM_EXTRACT_REMOVE_EXTRACTED']='no' + env['CM_EXTRACT_REMOVE_EXTRACTED'] = 'no' - if str(env.get('CM_DAE_EXTRACT_DOWNLOADED')).lower() in [ "yes", "1", "true" ]: - if (env.get('CM_EXTRACT_FINAL_ENV_NAME', '') == '') and (env.get('CM_DAE_FINAL_ENV_NAME', '') != ''): + if str(env.get('CM_DAE_EXTRACT_DOWNLOADED') + ).lower() in ["yes", "1", "true"]: + if (env.get('CM_EXTRACT_FINAL_ENV_NAME', '') == '') and ( + env.get('CM_DAE_FINAL_ENV_NAME', '') != ''): env['CM_EXTRACT_FINAL_ENV_NAME'] = env['CM_DAE_FINAL_ENV_NAME'] - return {'return':0} + return {'return': 0} + def postprocess(i): @@ -53,13 +59,15 @@ def postprocess(i): filepath = env.get('CM_DOWNLOAD_DOWNLOADED_PATH', '') if filepath == '': - return {'return':1, 'error': 'No extracted path set in "CM_EXTRACT_EXTRACTED_PATH"'} + return {'return': 1, + 'error': 'No extracted path set in "CM_EXTRACT_EXTRACTED_PATH"'} if not os.path.exists(filepath): - return {'return':1, 'error': 'Extracted path doesn\'t exist: {}'.format(filepath)} + return {'return': 1, + 'error': 'Extracted path doesn\'t exist: {}'.format(filepath)} if env.get('CM_DAE_FINAL_ENV_NAME'): env[env['CM_DAE_FINAL_ENV_NAME']] = filepath - env['CM_GET_DEPENDENT_CACHED_PATH'] = filepath + env['CM_GET_DEPENDENT_CACHED_PATH'] = filepath - return {'return':0} + return {'return': 0} diff --git a/script/download-torrent/customize.py b/script/download-torrent/customize.py index 52b57f253b..3b4cb4c41b 100644 --- a/script/download-torrent/customize.py +++ b/script/download-torrent/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -14,14 +15,17 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') if not env.get('CM_TORRENT_DOWNLOADED_FILE_NAME'): - return {'return':1, 'error': 'CM_TORRENT_DOWNLOADED_FILE_NAME is not set' } + return {'return': 1, 'error': 'CM_TORRENT_DOWNLOADED_FILE_NAME is not set'} + + return {'return': 0} - return {'return':0} def postprocess(i): env = i['env'] - torrent_downloaded_path = os.path.join(env['CM_TORRENT_DOWNLOADED_DIR'], env['CM_TORRENT_DOWNLOADED_NAME']) + torrent_downloaded_path = os.path.join( + env['CM_TORRENT_DOWNLOADED_DIR'], + env['CM_TORRENT_DOWNLOADED_NAME']) env['CM_TORRENT_DOWNLOADED_PATH'] = torrent_downloaded_path if 'CM_TORRENT_DOWNLOADED_PATH_ENV_KEY' in env: @@ -30,4 +34,4 @@ def postprocess(i): env['CM_GET_DEPENDENT_CACHED_PATH'] = torrent_downloaded_path - return {'return':0} + return {'return': 0} diff --git a/script/draw-graph-from-json-data/customize.py b/script/draw-graph-from-json-data/customize.py index a4a2aefaaa..db1f64b1f8 100644 --- a/script/draw-graph-from-json-data/customize.py +++ b/script/draw-graph-from-json-data/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -21,10 +22,11 @@ def preprocess(i): if env.get('CM_OUTPUT_MERMAID_PATH', '') != '': env['CM_RUN_CMD'] += f""" --output_mermaid {env['CM_OUTPUT_MERMAID_PATH']}""" - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/draw-graph-from-json-data/process-cm-deps.py b/script/draw-graph-from-json-data/process-cm-deps.py index c3f3747a6f..fe331a4a0a 100644 --- a/script/draw-graph-from-json-data/process-cm-deps.py +++ b/script/draw-graph-from-json-data/process-cm-deps.py @@ -4,6 +4,8 @@ import json # Function to parse the nested JSON structure + + def parse_json_to_edges(json_data): edges = [] for root_key, nodes in json_data.items(): @@ -12,6 +14,7 @@ def parse_json_to_edges(json_data): edges.append((node_details["parent"], node_key)) return edges + def generate_mermaid_output(json_data, mermaid_file="graph.mmd"): edges = parse_json_to_edges(json_data) @@ -20,7 +23,8 @@ def generate_mermaid_output(json_data, mermaid_file="graph.mmd"): # Add each edge in Mermaid syntax for parent, child in edges: - mermaid_lines.append(f""" {parent.replace(" ", "_")} --> {child.replace(" ", "_")}""") + mermaid_lines.append( + f""" {parent.replace(" ", "_")} --> {child.replace(" ", "_")}""") # Write to a Mermaid file with open(mermaid_file, "w") as f: @@ -29,7 +33,6 @@ def generate_mermaid_output(json_data, mermaid_file="graph.mmd"): print(f"Mermaid syntax saved to {mermaid_file}") - # Function to generate and visualize the graph def generate_graph_from_nested_json(json_data, output_image="graph.png"): # Parse the JSON to extract edges @@ -43,7 +46,7 @@ def generate_graph_from_nested_json(json_data, output_image="graph.png"): # Draw the graph using a spring layout for better visualization plt.figure(figsize=(30, 25)) - #pos = nx.spectral_layout(G, seed=42) # Seed for consistent layout + # pos = nx.spectral_layout(G, seed=42) # Seed for consistent layout pos = nx.shell_layout(G) # Seed for consistent layout nx.draw( G, @@ -62,22 +65,42 @@ def generate_graph_from_nested_json(json_data, output_image="graph.png"): # Save the visualization plt.savefig(output_image, format="png", dpi=300) print(f"Graph visualization saved as {output_image}") - #plt.show() + # plt.show() return G # Function to export the graph data + + def export_graph_data(graph, filename="graph.graphml"): nx.write_graphml(graph, filename) print(f"Graph data saved as {filename}") # Main function to handle argument parsing and processing + + def main(): - parser = argparse.ArgumentParser(description="Generate a graph from nested JSON input.") - parser.add_argument("json_file", type=str, help="Path to the JSON input file.") - parser.add_argument("--output_image", type=str, default="graph.png", help="Output image file for the graph visualization.") - parser.add_argument("--output_mermaid", type=str, default="graph.mmd", help="Output mermaid file for the graph data.") - parser.add_argument("--output_graphml", type=str, default="graph.graphml", help="Output GraphML file for the graph data.") + parser = argparse.ArgumentParser( + description="Generate a graph from nested JSON input.") + parser.add_argument( + "json_file", + type=str, + help="Path to the JSON input file.") + parser.add_argument( + "--output_image", + type=str, + default="graph.png", + help="Output image file for the graph visualization.") + parser.add_argument( + "--output_mermaid", + type=str, + default="graph.mmd", + help="Output mermaid file for the graph data.") + parser.add_argument( + "--output_graphml", + type=str, + default="graph.graphml", + help="Output GraphML file for the graph data.") args = parser.parse_args() @@ -86,12 +109,14 @@ def main(): json_data = json.load(f) # Generate the graph - G = generate_graph_from_nested_json(json_data, output_image=args.output_image) + G = generate_graph_from_nested_json( + json_data, output_image=args.output_image) generate_mermaid_output(json_data, mermaid_file="graph.mmd") # Export the graph data export_graph_data(G, filename=args.output_graphml) + if __name__ == "__main__": main() diff --git a/script/dump-pip-freeze/dump.py b/script/dump-pip-freeze/dump.py index 1d7f7ab853..c6d4dc2eaa 100644 --- a/script/dump-pip-freeze/dump.py +++ b/script/dump-pip-freeze/dump.py @@ -1,7 +1,9 @@ import os from pip._internal.operations import freeze -pip_freeze_out = os.environ.get('CM_DUMP_RAW_PIP_FREEZE_FILE_PATH', 'tmp-pip-freeze') +pip_freeze_out = os.environ.get( + 'CM_DUMP_RAW_PIP_FREEZE_FILE_PATH', + 'tmp-pip-freeze') if os.path.isfile(pip_freeze_out): os.remove(pip_freeze_out) @@ -12,10 +14,10 @@ try: for pkg in pkgs: - x+=pkg+'\n' -except: + x += pkg + '\n' +except BaseException: pass -if len(x)>0: +if len(x) > 0: with open(pip_freeze_out, "w") as f: f.write(x) diff --git a/script/fail/customize.py b/script/fail/customize.py index 855c39b5bb..4168a3b7a7 100644 --- a/script/fail/customize.py +++ b/script/fail/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -14,15 +15,16 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') # Checking conditions - if env.get('CM_FAIL_WINDOWS','').lower()=='true': + if env.get('CM_FAIL_WINDOWS', '').lower() == 'true': if os_info['platform'] == 'windows': - return {'return':1, 'error': 'CM detected fail condition: running on Windows'} + return {'return': 1, + 'error': 'CM detected fail condition: running on Windows'} + return {'return': 0} - return {'return':0} def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/flash-tinyml-binary/customize.py b/script/flash-tinyml-binary/customize.py index a2062be59c..bc0c3bd18c 100644 --- a/script/flash-tinyml-binary/customize.py +++ b/script/flash-tinyml-binary/customize.py @@ -1,19 +1,22 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] env = i['env'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} if 'CM_TINY_BUILD_DIR' not in env: - return {'return':1, 'error': 'Please set CM_TINY_BUILD_DIR to the build directory of the model'} - return {'return':0} + return { + 'return': 1, 'error': 'Please set CM_TINY_BUILD_DIR to the build directory of the model'} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/get-cache-dir/customize.py b/script/get-cache-dir/customize.py index 6e8a76460d..d1a3edd741 100644 --- a/script/get-cache-dir/customize.py +++ b/script/get-cache-dir/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,7 +14,8 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): @@ -26,4 +28,4 @@ def postprocess(i): env['CM_CACHE_DIR'] = cache_dir env['CM_GET_DEPENDENT_CACHED_PATH'] = cache_dir - return {'return':0} + return {'return': 0} diff --git a/script/get-cmsis_5/customize.py b/script/get-cmsis_5/customize.py index af9b13c464..9f9ebc556a 100644 --- a/script/get-cmsis_5/customize.py +++ b/script/get-cmsis_5/customize.py @@ -2,19 +2,21 @@ import os import shutil + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] if 'CM_GIT_DEPTH' not in env: env['CM_GIT_DEPTH'] = '' if 'CM_GIT_RECURSE_SUBMODULES' not in env: env['CM_GIT_RECURSE_SUBMODULES'] = '' - return {'return':0} + return {'return': 0} + def postprocess(i): @@ -22,4 +24,4 @@ def postprocess(i): state = i['state'] env['CMSIS_PATH'] = os.path.join(os.getcwd(), 'cmsis') - return {'return':0} + return {'return': 0} diff --git a/script/get-compiler-flags/customize.py b/script/get-compiler-flags/customize.py index 23ccbe6472..e9fc573015 100644 --- a/script/get-compiler-flags/customize.py +++ b/script/get-compiler-flags/customize.py @@ -2,6 +2,7 @@ import os import subprocess + def preprocess(i): os_info = i['os_info'] @@ -13,12 +14,13 @@ def preprocess(i): # TBD: add unified flags for Windows if os_info['platform'] == 'windows': - return {'return':0} + return {'return': 0} - if env.get("CM_FAST_COMPILATION") in [ "yes", "on", "1" ]: + if env.get("CM_FAST_COMPILATION") in ["yes", "on", "1"]: DEFAULT_COMPILER_FLAGS = env.get("CM_COMPILER_FLAGS_FAST", "-O3") - DEFAULT_LINKER_FLAGS = env.get("CM_LINKER_FLAGS_FAST", "-O3") # -flto") - this flag is not always available - elif env.get("CM_DEBUG_COMPILATION") in ["yes", "on", "1" ]: + # -flto") - this flag is not always available + DEFAULT_LINKER_FLAGS = env.get("CM_LINKER_FLAGS_FAST", "-O3") + elif env.get("CM_DEBUG_COMPILATION") in ["yes", "on", "1"]: DEFAULT_COMPILER_FLAGS = env.get("CM_COMPILER_FLAGS_DEBUG", "-O0") DEFAULT_LINKER_FLAGS = env.get("CM_LINKER_FLAGS_DEBUG", "-O0") else: @@ -60,4 +62,4 @@ def preprocess(i): # if int(env['CM_HOST_CPU_FAMILY']) >= 0: # env['+ CFLAGS'] += ["-march=znver2", "-flto"] - return {'return':0} + return {'return': 0} diff --git a/script/get-compiler-rust/customize.py b/script/get-compiler-rust/customize.py index cd42edf7bf..2a1b202f81 100644 --- a/script/get-compiler-rust/customize.py +++ b/script/get-compiler-rust/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,13 +14,14 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - rust_path = os.path.join(os.path.expanduser('~'),".cargo", "bin") - env['+PATH'] = [ rust_path ] + rust_path = os.path.join(os.path.expanduser('~'), ".cargo", "bin") + env['+PATH'] = [rust_path] - return {'return':0} + return {'return': 0} diff --git a/script/get-croissant/customize.py b/script/get-croissant/customize.py index 1ced8a4846..afa8214659 100644 --- a/script/get-croissant/customize.py +++ b/script/get-croissant/customize.py @@ -1,16 +1,17 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] - return {'return':0} + return {'return': 0} + def postprocess(i): os_info = i['os_info'] env = i['env'] - - return {'return':0} + return {'return': 0} diff --git a/script/get-dataset-cifar10/customize.py b/script/get-dataset-cifar10/customize.py index b624357402..d000dc407d 100644 --- a/script/get-dataset-cifar10/customize.py +++ b/script/get-dataset-cifar10/customize.py @@ -2,15 +2,17 @@ import os import shutil + def preprocess(i): env = i['env'] return {'return': 0} + def postprocess(i): env = i['env'] - variation_tags = i.get('variation_tags',[]) + variation_tags = i.get('variation_tags', []) return {'return': 0} diff --git a/script/get-dataset-cnndm/customize.py b/script/get-dataset-cnndm/customize.py index 27363d8000..25966d8505 100644 --- a/script/get-dataset-cnndm/customize.py +++ b/script/get-dataset-cnndm/customize.py @@ -2,6 +2,7 @@ import os import shutil + def preprocess(i): env = i['env'] @@ -9,21 +10,27 @@ def preprocess(i): if env.get('CM_CNNDM_INTEL_VARIATION', '') == 'yes': i['run_script_input']['script_name'] = "run-intel" else: - print("Using MLCommons Inference source from '" + env['CM_MLPERF_INFERENCE_SOURCE'] +"'") + print("Using MLCommons Inference source from '" + + env['CM_MLPERF_INFERENCE_SOURCE'] + "'") return {'return': 0} + def postprocess(i): env = i['env'] - if env.get('CM_DATASET_CALIBRATION','') == "no": + if env.get('CM_DATASET_CALIBRATION', '') == "no": env['CM_DATASET_PATH'] = os.path.join(os.getcwd(), 'install') - env['CM_DATASET_EVAL_PATH'] = os.path.join(os.getcwd(), 'install', 'cnn_eval.json') - env['CM_DATASET_CNNDM_EVAL_PATH'] = os.path.join(os.getcwd(), 'install', 'cnn_eval.json') + env['CM_DATASET_EVAL_PATH'] = os.path.join( + os.getcwd(), 'install', 'cnn_eval.json') + env['CM_DATASET_CNNDM_EVAL_PATH'] = os.path.join( + os.getcwd(), 'install', 'cnn_eval.json') env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_DATASET_PATH'] else: - env['CM_CALIBRATION_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'cnn_dailymail_calibration.json') - env['CM_CALIBRATION_DATASET_CNNDM_PATH'] = os.path.join(os.getcwd(), 'install', 'cnn_dailymail_calibration.json') + env['CM_CALIBRATION_DATASET_PATH'] = os.path.join( + os.getcwd(), 'install', 'cnn_dailymail_calibration.json') + env['CM_CALIBRATION_DATASET_CNNDM_PATH'] = os.path.join( + os.getcwd(), 'install', 'cnn_dailymail_calibration.json') env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_CALIBRATION_DATASET_PATH'] return {'return': 0} diff --git a/script/get-dataset-imagenet-helper/imagenet_helper/__init__.py b/script/get-dataset-imagenet-helper/imagenet_helper/__init__.py index d28c502fc5..aa90deefd8 100644 --- a/script/get-dataset-imagenet-helper/imagenet_helper/__init__.py +++ b/script/get-dataset-imagenet-helper/imagenet_helper/__init__.py @@ -4,65 +4,84 @@ import numpy as np -## Processing in batches: +# Processing in batches: # -BATCH_SIZE = int(os.getenv('CM_BATCH_SIZE', 1)) +BATCH_SIZE = int(os.getenv('CM_BATCH_SIZE', 1)) -## Model properties: +# Model properties: # -MODEL_IMAGE_HEIGHT = int(os.getenv('CM_ML_MODEL_IMAGE_HEIGHT', - os.getenv('CM_ONNX_MODEL_IMAGE_HEIGHT', - os.getenv('CM_TENSORFLOW_MODEL_IMAGE_HEIGHT', - '')))) -MODEL_IMAGE_WIDTH = int(os.getenv('CM_ML_MODEL_IMAGE_WIDTH', - os.getenv('CM_ONNX_MODEL_IMAGE_WIDTH', - os.getenv('CM_TENSORFLOW_MODEL_IMAGE_WIDTH', - '')))) -MODEL_IMAGE_CHANNELS = int(os.getenv('CM_ML_MODEL_IMAGE_CHANNELS', 3)) -MODEL_DATA_LAYOUT = os.getenv('CM_ML_MODEL_DATA_LAYOUT', 'NCHW') -MODEL_COLOURS_BGR = os.getenv('CM_ML_MODEL_COLOUR_CHANNELS_BGR', 'NO') in ('YES', 'yes', 'ON', 'on', '1') -MODEL_INPUT_DATA_TYPE = os.getenv('CM_ML_MODEL_INPUT_DATA_TYPE', 'float32') -MODEL_DATA_TYPE = os.getenv('CM_ML_MODEL_DATA_TYPE', '(unknown)') -MODEL_USE_DLA = os.getenv('CM_ML_MODEL_USE_DLA', 'NO') in ('YES', 'yes', 'ON', 'on', '1') -MODEL_MAX_BATCH_SIZE = int(os.getenv('CM_ML_MODEL_MAX_BATCH_SIZE', BATCH_SIZE)) - - -## Internal processing: +MODEL_IMAGE_HEIGHT = int(os.getenv('CM_ML_MODEL_IMAGE_HEIGHT', + os.getenv('CM_ONNX_MODEL_IMAGE_HEIGHT', + os.getenv('CM_TENSORFLOW_MODEL_IMAGE_HEIGHT', + '')))) +MODEL_IMAGE_WIDTH = int(os.getenv('CM_ML_MODEL_IMAGE_WIDTH', + os.getenv('CM_ONNX_MODEL_IMAGE_WIDTH', + os.getenv('CM_TENSORFLOW_MODEL_IMAGE_WIDTH', + '')))) +MODEL_IMAGE_CHANNELS = int(os.getenv('CM_ML_MODEL_IMAGE_CHANNELS', 3)) +MODEL_DATA_LAYOUT = os.getenv('CM_ML_MODEL_DATA_LAYOUT', 'NCHW') +MODEL_COLOURS_BGR = os.getenv( + 'CM_ML_MODEL_COLOUR_CHANNELS_BGR', 'NO') in ( + 'YES', 'yes', 'ON', 'on', '1') +MODEL_INPUT_DATA_TYPE = os.getenv('CM_ML_MODEL_INPUT_DATA_TYPE', 'float32') +MODEL_DATA_TYPE = os.getenv('CM_ML_MODEL_DATA_TYPE', '(unknown)') +MODEL_USE_DLA = os.getenv( + 'CM_ML_MODEL_USE_DLA', + 'NO') in ( + 'YES', + 'yes', + 'ON', + 'on', + '1') +MODEL_MAX_BATCH_SIZE = int(os.getenv('CM_ML_MODEL_MAX_BATCH_SIZE', BATCH_SIZE)) + + +# Internal processing: # -INTERMEDIATE_DATA_TYPE = np.float32 # default for internal conversion -#INTERMEDIATE_DATA_TYPE = np.int8 # affects the accuracy a bit +INTERMEDIATE_DATA_TYPE = np.float32 # default for internal conversion +# INTERMEDIATE_DATA_TYPE = np.int8 # affects the accuracy a bit -## Image normalization: +# Image normalization: # -MODEL_NORMALIZE_DATA = os.getenv('CM_ML_MODEL_NORMALIZE_DATA') in ('YES', 'yes', 'ON', 'on', '1') -MODEL_NORMALIZE_LOWER = float(os.getenv('CM_ML_MODEL_NORMALIZE_LOWER', -1.0)) -MODEL_NORMALIZE_UPPER = float(os.getenv('CM_ML_MODEL_NORMALIZE_UPPER', 1.0)) -SUBTRACT_MEAN = os.getenv('CM_ML_MODEL_SUBTRACT_MEANS', 'YES') in ('YES', 'yes', 'ON', 'on', '1') -GIVEN_CHANNEL_MEANS = os.getenv('CM_ML_MODEL_GIVEN_CHANNEL_MEANS', '') +MODEL_NORMALIZE_DATA = os.getenv('CM_ML_MODEL_NORMALIZE_DATA') in ( + 'YES', 'yes', 'ON', 'on', '1') +MODEL_NORMALIZE_LOWER = float(os.getenv('CM_ML_MODEL_NORMALIZE_LOWER', -1.0)) +MODEL_NORMALIZE_UPPER = float(os.getenv('CM_ML_MODEL_NORMALIZE_UPPER', 1.0)) +SUBTRACT_MEAN = os.getenv( + 'CM_ML_MODEL_SUBTRACT_MEANS', 'YES') in ( + 'YES', 'yes', 'ON', 'on', '1') +GIVEN_CHANNEL_MEANS = os.getenv('CM_ML_MODEL_GIVEN_CHANNEL_MEANS', '') if GIVEN_CHANNEL_MEANS: - GIVEN_CHANNEL_MEANS = np.fromstring(GIVEN_CHANNEL_MEANS, dtype=np.float32, sep=' ').astype(INTERMEDIATE_DATA_TYPE) + GIVEN_CHANNEL_MEANS = np.fromstring( + GIVEN_CHANNEL_MEANS, + dtype=np.float32, + sep=' ').astype(INTERMEDIATE_DATA_TYPE) if MODEL_COLOURS_BGR: - GIVEN_CHANNEL_MEANS = GIVEN_CHANNEL_MEANS[::-1] # swapping Red and Blue colour channels + # swapping Red and Blue colour channels + GIVEN_CHANNEL_MEANS = GIVEN_CHANNEL_MEANS[::-1] -GIVEN_CHANNEL_STDS = os.getenv('CM_ML_MODEL_GIVEN_CHANNEL_STDS', '') +GIVEN_CHANNEL_STDS = os.getenv('CM_ML_MODEL_GIVEN_CHANNEL_STDS', '') if GIVEN_CHANNEL_STDS: - GIVEN_CHANNEL_STDS = np.fromstring(GIVEN_CHANNEL_STDS, dtype=np.float32, sep=' ').astype(INTERMEDIATE_DATA_TYPE) + GIVEN_CHANNEL_STDS = np.fromstring( + GIVEN_CHANNEL_STDS, + dtype=np.float32, + sep=' ').astype(INTERMEDIATE_DATA_TYPE) if MODEL_COLOURS_BGR: - GIVEN_CHANNEL_STDS = GIVEN_CHANNEL_STDS[::-1] # swapping Red and Blue colour channels + # swapping Red and Blue colour channels + GIVEN_CHANNEL_STDS = GIVEN_CHANNEL_STDS[::-1] - -## ImageNet dataset properties: +# ImageNet dataset properties: # -LABELS_PATH = os.environ['CM_CAFFE_IMAGENET_SYNSET_WORDS_TXT'] +LABELS_PATH = os.environ['CM_CAFFE_IMAGENET_SYNSET_WORDS_TXT'] -## Preprocessed input images' properties: +# Preprocessed input images' properties: # -IMAGE_DIR = os.getenv('CM_DATASET_PREPROCESSED_PATH') -IMAGE_DATA_TYPE = os.getenv('CM_DATASET_PREPROCESSED_DATA_TYPE', 'float32') +IMAGE_DIR = os.getenv('CM_DATASET_PREPROCESSED_PATH') +IMAGE_DATA_TYPE = os.getenv('CM_DATASET_PREPROCESSED_DATA_TYPE', 'float32') def load_labels(labels_filepath): @@ -72,6 +91,7 @@ def load_labels(labels_filepath): my_labels.append(l.strip()) return my_labels + class_labels = load_labels(LABELS_PATH) @@ -82,41 +102,45 @@ def load_labels(labels_filepath): if image_file.endswith('.npy'): image_list.append(image_file) + def load_image_by_index_and_normalize(image_index): img_file = os.path.join(IMAGE_DIR, image_list[image_index]) img = np.fromfile(img_file, np.dtype(IMAGE_DATA_TYPE)) - #img = img.reshape((1,MODEL_IMAGE_HEIGHT, MODEL_IMAGE_WIDTH, 3)) - img.resize(224*224*3) - img = img.reshape((MODEL_IMAGE_HEIGHT, MODEL_IMAGE_WIDTH, MODEL_IMAGE_CHANNELS)) + # img = img.reshape((1,MODEL_IMAGE_HEIGHT, MODEL_IMAGE_WIDTH, 3)) + img.resize(224 * 224 * 3) + img = img.reshape( + (MODEL_IMAGE_HEIGHT, + MODEL_IMAGE_WIDTH, + MODEL_IMAGE_CHANNELS)) if MODEL_COLOURS_BGR: - img = img[...,::-1] # swapping Red and Blue colour channels + img = img[..., ::-1] # swapping Red and Blue colour channels if IMAGE_DATA_TYPE != 'float32': img = img.astype(np.float32) # Normalize if MODEL_NORMALIZE_DATA: - img /= (255.0/(MODEL_NORMALIZE_UPPER-MODEL_NORMALIZE_LOWER)) + img /= (255.0 / (MODEL_NORMALIZE_UPPER - MODEL_NORMALIZE_LOWER)) img += MODEL_NORMALIZE_LOWER # Subtract mean value if len(GIVEN_CHANNEL_MEANS): img -= GIVEN_CHANNEL_MEANS elif SUBTRACT_MEAN: - img -= np.mean(img, axis=(0,1), keepdims=True) + img -= np.mean(img, axis=(0, 1), keepdims=True) if len(GIVEN_CHANNEL_STDS): img /= GIVEN_CHANNEL_STDS - if MODEL_INPUT_DATA_TYPE == 'int8' or INTERMEDIATE_DATA_TYPE==np.int8: + if MODEL_INPUT_DATA_TYPE == 'int8' or INTERMEDIATE_DATA_TYPE == np.int8: img = np.clip(img, -128, 127).astype(INTERMEDIATE_DATA_TYPE) if MODEL_DATA_LAYOUT == 'NCHW': - img = img.transpose(2,0,1) + img = img.transpose(2, 0, 1) elif MODEL_DATA_LAYOUT == 'CHW4': - img = np.pad(img, ((0,0), (0,0), (0,1)), 'constant') + img = np.pad(img, ((0, 0), (0, 0), (0, 1)), 'constant') # Add img to batch return img.astype(MODEL_INPUT_DATA_TYPE) @@ -127,13 +151,15 @@ def load_preprocessed_batch(image_list, image_index): for in_batch_idx in range(BATCH_SIZE): img = load_image_by_index_and_normalize(image_index) if batch_data is None: - batch_data = np.empty( (BATCH_SIZE, *img.shape), dtype=MODEL_INPUT_DATA_TYPE) + batch_data = np.empty( + (BATCH_SIZE, *img.shape), dtype=MODEL_INPUT_DATA_TYPE) batch_data[in_batch_idx] = img image_index += 1 - #print('Data shape: {}'.format(batch_data.shape)) + # print('Data shape: {}'.format(batch_data.shape)) - if MODEL_USE_DLA and MODEL_MAX_BATCH_SIZE>len(batch_data): - return np.pad(batch_data, ((0,MODEL_MAX_BATCH_SIZE-len(batch_data)), (0,0), (0,0), (0,0)), 'constant'), image_index + if MODEL_USE_DLA and MODEL_MAX_BATCH_SIZE > len(batch_data): + return np.pad(batch_data, ((0, MODEL_MAX_BATCH_SIZE - len(batch_data)), + (0, 0), (0, 0), (0, 0)), 'constant'), image_index else: return batch_data, image_index diff --git a/script/get-dataset-imagenet-train/customize.py b/script/get-dataset-imagenet-train/customize.py index 2eba2b9b15..b1f8aea1ea 100644 --- a/script/get-dataset-imagenet-train/customize.py +++ b/script/get-dataset-imagenet-train/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -10,7 +11,7 @@ def preprocess(i): meta = i['meta'] os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':0} + return {'return': 0} env['CM_DATASET_IMAGENET_TRAIN_REQUIRE_DAE'] = 'no' @@ -23,30 +24,31 @@ def preprocess(i): env['CM_DAE_TORRENT_PATH'] = path env['CM_DATASET_IMAGENET_VAL_REQUIRE_DAE'] = 'yes' - return {'return':0} + return {'return': 0} else: - return {'return':1, 'error':'Please rerun the last CM command with --env.IMAGENET_TRAIN_PATH={path the folder containing full ImageNet training images} or envoke cm run script "get train dataset imagenet" --input={path to the folder containing ImageNet training images}'} - + return {'return': 1, 'error': 'Please rerun the last CM command with --env.IMAGENET_TRAIN_PATH={path the folder containing full ImageNet training images} or envoke cm run script "get train dataset imagenet" --input={path to the folder containing ImageNet training images}'} elif not os.path.isdir(path): if path.endswith(".tar"): - #env['CM_DAE_FILEPATH'] = path + # env['CM_DAE_FILEPATH'] = path env['CM_EXTRACT_FILEPATH'] = path env['CM_DAE_ONLY_EXTRACT'] = 'yes' - return {'return':0} + return {'return': 0} else: - return {'return':1, 'error':'Path {} doesn\'t exist'.format(path)} + return {'return': 1, + 'error': 'Path {} doesn\'t exist'.format(path)} else: env['CM_EXTRACT_EXTRACTED_PATH'] = path - return {'return':0} + return {'return': 0} + def postprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':0} + return {'return': 0} env = i['env'] @@ -55,12 +57,13 @@ def postprocess(i): path_tar = os.path.join(path, 'n01440764.tar') if not os.path.isfile(path_tar): - return {'return':1, 'error':'ImageNet file {} not found'.format(path_tar)} + return {'return': 1, + 'error': 'ImageNet file {} not found'.format(path_tar)} env['CM_DATASET_PATH'] = path env['CM_DATASET_IMAGENET_PATH'] = path env['CM_DATASET_IMAGENET_TRAIN_PATH'] = path - env['CM_GET_DEPENDENT_CACHED_PATH'] = path + env['CM_GET_DEPENDENT_CACHED_PATH'] = path - return {'return':0} + return {'return': 0} diff --git a/script/get-dataset-kits19/customize.py b/script/get-dataset-kits19/customize.py index 97c583e9d5..bf5462d9b4 100644 --- a/script/get-dataset-kits19/customize.py +++ b/script/get-dataset-kits19/customize.py @@ -2,18 +2,20 @@ import os import shutil + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] meta = i['meta'] - if not env.get('CM_GIT_CHECKOUT',''): - return {'return':1, 'error': 'Please provide a valid CM_GIT_SHA inside the custom variation of _cm.json'} + if not env.get('CM_GIT_CHECKOUT', ''): + return { + 'return': 1, 'error': 'Please provide a valid CM_GIT_SHA inside the custom variation of _cm.json'} if 'CM_GIT_DEPTH' not in env: env['CM_GIT_DEPTH'] = '' @@ -21,13 +23,13 @@ def preprocess(i): if 'CM_GIT_RECURSE_SUBMODULES' not in env: env['CM_GIT_RECURSE_SUBMODULES'] = '' - need_version = env.get('CM_VERSION','') + need_version = env.get('CM_VERSION', '') versions = meta['versions'] - if need_version!='' and not need_version in versions: + if need_version != '' and not need_version in versions: env['CM_GIT_CHECKOUT'] = need_version - return {'return':0} + return {'return': 0} def postprocess(i): @@ -36,4 +38,4 @@ def postprocess(i): env['CM_DATASET_PATH'] = os.path.join(os.getcwd(), 'kits19', 'data') state = i['state'] - return {'return':0} + return {'return': 0} diff --git a/script/get-dataset-librispeech/customize.py b/script/get-dataset-librispeech/customize.py index 85ec8e43c9..0da64539eb 100644 --- a/script/get-dataset-librispeech/customize.py +++ b/script/get-dataset-librispeech/customize.py @@ -1,19 +1,22 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] env = i['env'] - return {'return':0} + return {'return': 0} def postprocess(i): env = i['env'] - folder_name = env['CM_DATASET_ARCHIVE'].split(".")[0] - env['CM_DATASET_LIBRISPEECH_PATH'] = os.path.join(os.getcwd(), "LibriSpeech", folder_name) - env['CM_DATASET_PATH'] = os.path.join(os.getcwd(), "LibriSpeech", folder_name) + folder_name = env['CM_DATASET_ARCHIVE'].split(".")[0] + env['CM_DATASET_LIBRISPEECH_PATH'] = os.path.join( + os.getcwd(), "LibriSpeech", folder_name) + env['CM_DATASET_PATH'] = os.path.join( + os.getcwd(), "LibriSpeech", folder_name) - return {'return':0} + return {'return': 0} diff --git a/script/get-dataset-openimages-calibration/customize.py b/script/get-dataset-openimages-calibration/customize.py index 71e1a646d4..032065bc86 100644 --- a/script/get-dataset-openimages-calibration/customize.py +++ b/script/get-dataset-openimages-calibration/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -15,13 +16,15 @@ def preprocess(i): if env.get("CM_CALIBRATE_FILTER", "") == "yes": i['run_script_input']['script_name'] = "run-filter" - env['CM_MLPERF_OPENIMAGES_CALIBRATION_FILTERED_LIST'] = os.path.join(os.getcwd(), "filtered.txt") + env['CM_MLPERF_OPENIMAGES_CALIBRATION_FILTERED_LIST'] = os.path.join( + os.getcwd(), "filtered.txt") env['CM_MLPERF_OPENIMAGES_CALIBRATION_LIST_FILE_WITH_PATH'] = env['CM_MLPERF_OPENIMAGES_CALIBRATION_FILTERED_LIST'] - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/get-dataset-openimages-calibration/filter.py b/script/get-dataset-openimages-calibration/filter.py index 81b768249c..d8d2638b57 100644 --- a/script/get-dataset-openimages-calibration/filter.py +++ b/script/get-dataset-openimages-calibration/filter.py @@ -5,7 +5,7 @@ with open(sys.argv[1], "r") as f: data = json.load(f) -images= {} +images = {} for image in data['images']: images[image['id']] = image images[image['id']]['num_boxes'] = 0 @@ -13,8 +13,13 @@ annots = data['annotations'] for box in annots: imageid = box['image_id'] - images[imageid]['num_boxes']+=1 + images[imageid]['num_boxes'] += 1 -sorted_image_data = sorted(data['images'], key=lambda x: x['num_boxes'], reverse= os.environ.get('CM_CALIBRATION_FILTER_ORDER_BY_NUM_BOXES_ASC', '') == "yes") +sorted_image_data = sorted( + data['images'], + key=lambda x: x['num_boxes'], + reverse=os.environ.get( + 'CM_CALIBRATION_FILTER_ORDER_BY_NUM_BOXES_ASC', + '') == "yes") for image in data['images']: print(image['file_name']) diff --git a/script/get-dataset-openimages/customize.py b/script/get-dataset-openimages/customize.py index 84d1f52312..ae53a85f5c 100644 --- a/script/get-dataset-openimages/customize.py +++ b/script/get-dataset-openimages/customize.py @@ -2,86 +2,100 @@ import os import shutil + def preprocess(i): os_info = i['os_info'] env = i['env'] - print ("") - print ("Using MLCommons Inference source from '" + env['CM_MLPERF_INFERENCE_SOURCE'] +"'") - print ("") + print("") + print("Using MLCommons Inference source from '" + + env['CM_MLPERF_INFERENCE_SOURCE'] + "'") + print("") if os_info['platform'] == 'windows': - MLPERF_CLASSES=['Airplane','Antelope','Apple','Backpack','Balloon','Banana', - 'Barrel','Baseball bat','Baseball glove','Bee','Beer','Bench','Bicycle', - 'Bicycle helmet','Bicycle wheel','Billboard','Book','Bookcase','Boot', - 'Bottle','Bowl','Bowling equipment','Box','Boy','Brassiere','Bread', - 'Broccoli','Bronze sculpture','Bull','Bus','Bust','Butterfly','Cabinetry', - 'Cake','Camel','Camera','Candle','Candy','Cannon','Canoe','Carrot','Cart', - 'Castle','Cat','Cattle','Cello','Chair','Cheese','Chest of drawers','Chicken', - 'Christmas tree','Coat','Cocktail','Coffee','Coffee cup','Coffee table','Coin', - 'Common sunflower','Computer keyboard','Computer monitor','Convenience store', - 'Cookie','Countertop','Cowboy hat','Crab','Crocodile','Cucumber','Cupboard', - 'Curtain','Deer','Desk','Dinosaur','Dog','Doll','Dolphin','Door','Dragonfly', - 'Drawer','Dress','Drum','Duck','Eagle','Earrings','Egg (Food)','Elephant', - 'Falcon','Fedora','Flag','Flowerpot','Football','Football helmet','Fork', - 'Fountain','French fries','French horn','Frog','Giraffe','Girl','Glasses', - 'Goat','Goggles','Goldfish','Gondola','Goose','Grape','Grapefruit','Guitar', - 'Hamburger','Handbag','Harbor seal','Headphones','Helicopter','High heels', - 'Hiking equipment','Horse','House','Houseplant','Human arm','Human beard', - 'Human body','Human ear','Human eye','Human face','Human foot','Human hair', - 'Human hand','Human head','Human leg','Human mouth','Human nose','Ice cream', - 'Jacket','Jeans','Jellyfish','Juice','Kitchen & dining room table','Kite', - 'Lamp','Lantern','Laptop','Lavender (Plant)','Lemon','Light bulb','Lighthouse', - 'Lily','Lion','Lipstick','Lizard','Man','Maple','Microphone','Mirror', - 'Mixing bowl','Mobile phone','Monkey','Motorcycle','Muffin','Mug','Mule', - 'Mushroom','Musical keyboard','Necklace','Nightstand','Office building', - 'Orange','Owl','Oyster','Paddle','Palm tree','Parachute','Parrot','Pen', - 'Penguin','Personal flotation device','Piano','Picture frame','Pig','Pillow', - 'Pizza','Plate','Platter','Porch','Poster','Pumpkin','Rabbit','Rifle', - 'Roller skates','Rose','Salad','Sandal','Saucer','Saxophone','Scarf','Sea lion', - 'Sea turtle','Sheep','Shelf','Shirt','Shorts','Shrimp','Sink','Skateboard', - 'Ski','Skull','Skyscraper','Snake','Sock','Sofa bed','Sparrow','Spider','Spoon', - 'Sports uniform','Squirrel','Stairs','Stool','Strawberry','Street light', - 'Studio couch','Suit','Sun hat','Sunglasses','Surfboard','Sushi','Swan', - 'Swimming pool','Swimwear','Tank','Tap','Taxi','Tea','Teddy bear','Television', - 'Tent','Tie','Tiger','Tin can','Tire','Toilet','Tomato','Tortoise','Tower', - 'Traffic light','Train','Tripod','Truck','Trumpet','Umbrella','Van','Vase', - 'Vehicle registration plate','Violin','Wall clock','Waste container','Watch', - 'Whale','Wheel','Wheelchair','Whiteboard','Window','Wine','Wine glass','Woman', - 'Zebra','Zucchini'] + MLPERF_CLASSES = ['Airplane', 'Antelope', 'Apple', 'Backpack', 'Balloon', 'Banana', + 'Barrel', 'Baseball bat', 'Baseball glove', 'Bee', 'Beer', 'Bench', 'Bicycle', + 'Bicycle helmet', 'Bicycle wheel', 'Billboard', 'Book', 'Bookcase', 'Boot', + 'Bottle', 'Bowl', 'Bowling equipment', 'Box', 'Boy', 'Brassiere', 'Bread', + 'Broccoli', 'Bronze sculpture', 'Bull', 'Bus', 'Bust', 'Butterfly', 'Cabinetry', + 'Cake', 'Camel', 'Camera', 'Candle', 'Candy', 'Cannon', 'Canoe', 'Carrot', 'Cart', + 'Castle', 'Cat', 'Cattle', 'Cello', 'Chair', 'Cheese', 'Chest of drawers', 'Chicken', + 'Christmas tree', 'Coat', 'Cocktail', 'Coffee', 'Coffee cup', 'Coffee table', 'Coin', + 'Common sunflower', 'Computer keyboard', 'Computer monitor', 'Convenience store', + 'Cookie', 'Countertop', 'Cowboy hat', 'Crab', 'Crocodile', 'Cucumber', 'Cupboard', + 'Curtain', 'Deer', 'Desk', 'Dinosaur', 'Dog', 'Doll', 'Dolphin', 'Door', 'Dragonfly', + 'Drawer', 'Dress', 'Drum', 'Duck', 'Eagle', 'Earrings', 'Egg (Food)', 'Elephant', + 'Falcon', 'Fedora', 'Flag', 'Flowerpot', 'Football', 'Football helmet', 'Fork', + 'Fountain', 'French fries', 'French horn', 'Frog', 'Giraffe', 'Girl', 'Glasses', + 'Goat', 'Goggles', 'Goldfish', 'Gondola', 'Goose', 'Grape', 'Grapefruit', 'Guitar', + 'Hamburger', 'Handbag', 'Harbor seal', 'Headphones', 'Helicopter', 'High heels', + 'Hiking equipment', 'Horse', 'House', 'Houseplant', 'Human arm', 'Human beard', + 'Human body', 'Human ear', 'Human eye', 'Human face', 'Human foot', 'Human hair', + 'Human hand', 'Human head', 'Human leg', 'Human mouth', 'Human nose', 'Ice cream', + 'Jacket', 'Jeans', 'Jellyfish', 'Juice', 'Kitchen & dining room table', 'Kite', + 'Lamp', 'Lantern', 'Laptop', 'Lavender (Plant)', 'Lemon', 'Light bulb', 'Lighthouse', + 'Lily', 'Lion', 'Lipstick', 'Lizard', 'Man', 'Maple', 'Microphone', 'Mirror', + 'Mixing bowl', 'Mobile phone', 'Monkey', 'Motorcycle', 'Muffin', 'Mug', 'Mule', + 'Mushroom', 'Musical keyboard', 'Necklace', 'Nightstand', 'Office building', + 'Orange', 'Owl', 'Oyster', 'Paddle', 'Palm tree', 'Parachute', 'Parrot', 'Pen', + 'Penguin', 'Personal flotation device', 'Piano', 'Picture frame', 'Pig', 'Pillow', + 'Pizza', 'Plate', 'Platter', 'Porch', 'Poster', 'Pumpkin', 'Rabbit', 'Rifle', + 'Roller skates', 'Rose', 'Salad', 'Sandal', 'Saucer', 'Saxophone', 'Scarf', 'Sea lion', + 'Sea turtle', 'Sheep', 'Shelf', 'Shirt', 'Shorts', 'Shrimp', 'Sink', 'Skateboard', + 'Ski', 'Skull', 'Skyscraper', 'Snake', 'Sock', 'Sofa bed', 'Sparrow', 'Spider', 'Spoon', + 'Sports uniform', 'Squirrel', 'Stairs', 'Stool', 'Strawberry', 'Street light', + 'Studio couch', 'Suit', 'Sun hat', 'Sunglasses', 'Surfboard', 'Sushi', 'Swan', + 'Swimming pool', 'Swimwear', 'Tank', 'Tap', 'Taxi', 'Tea', 'Teddy bear', 'Television', + 'Tent', 'Tie', 'Tiger', 'Tin can', 'Tire', 'Toilet', 'Tomato', 'Tortoise', 'Tower', + 'Traffic light', 'Train', 'Tripod', 'Truck', 'Trumpet', 'Umbrella', 'Van', 'Vase', + 'Vehicle registration plate', 'Violin', 'Wall clock', 'Waste container', 'Watch', + 'Whale', 'Wheel', 'Wheelchair', 'Whiteboard', 'Window', 'Wine', 'Wine glass', 'Woman', + 'Zebra', 'Zucchini'] x = '' for v in MLPERF_CLASSES: - if x!='': x+=' ' - x+='"'+v+'"' - env['CM_DATASET_OPENIMAGES_CLASSES']=x + if x != '': + x += ' ' + x += '"' + v + '"' + env['CM_DATASET_OPENIMAGES_CLASSES'] = x return {'return': 0} + def postprocess(i): env = i['env'] - env['CM_DATASET_ANNOTATIONS_DIR_PATH'] = os.path.join(os.getcwd(), 'install', 'annotations') + env['CM_DATASET_ANNOTATIONS_DIR_PATH'] = os.path.join( + os.getcwd(), 'install', 'annotations') - if env.get('CM_DATASET_CALIBRATION','') == "no": + if env.get('CM_DATASET_CALIBRATION', '') == "no": env['CM_DATASET_PATH_ROOT'] = os.path.join(os.getcwd(), 'install') - env['CM_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'validation', 'data') - annotations_file_path = os.path.join(env['CM_DATASET_ANNOTATIONS_DIR_PATH'], "openimages-mlperf.json") + env['CM_DATASET_PATH'] = os.path.join( + os.getcwd(), 'install', 'validation', 'data') + annotations_file_path = os.path.join( + env['CM_DATASET_ANNOTATIONS_DIR_PATH'], + "openimages-mlperf.json") env['CM_DATASET_VALIDATION_ANNOTATIONS_FILE_PATH'] = annotations_file_path env['CM_DATASET_ANNOTATIONS_FILE_PATH'] = annotations_file_path env['CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH'] = annotations_file_path - if env.get("CM_DATASET_OPENIMAGES_CUSTOM_ANNOTATIONS",'') == "yes": + if env.get("CM_DATASET_OPENIMAGES_CUSTOM_ANNOTATIONS", '') == "yes": annotations_file_src = env['CM_DATASET_OPENIMAGES_ANNOTATIONS_FILE_PATH'] - shutil.copy(annotations_file_src, env['CM_DATASET_ANNOTATIONS_DIR_PATH']) + shutil.copy( + annotations_file_src, + env['CM_DATASET_ANNOTATIONS_DIR_PATH']) env['CM_DATASET_OPENIMAGES_PATH'] = env['CM_DATASET_PATH'] env['CM_DATASET_OPENIMAGES_PATH_ROOT'] = env['CM_DATASET_PATH_ROOT'] else: - env['CM_CALIBRATION_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'calibration', 'data') - env['CM_OPENIMAGES_CALIBRATION_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'calibration', 'data') - env['CM_CALIBRATION_DATASET_PATH_ROOT'] = os.path.join(os.getcwd(), 'install') - annotations_file_path = os.path.join(env['CM_DATASET_ANNOTATIONS_DIR_PATH'], "openimages-calibration-mlperf.json") + env['CM_CALIBRATION_DATASET_PATH'] = os.path.join( + os.getcwd(), 'install', 'calibration', 'data') + env['CM_OPENIMAGES_CALIBRATION_DATASET_PATH'] = os.path.join( + os.getcwd(), 'install', 'calibration', 'data') + env['CM_CALIBRATION_DATASET_PATH_ROOT'] = os.path.join( + os.getcwd(), 'install') + annotations_file_path = os.path.join( + env['CM_DATASET_ANNOTATIONS_DIR_PATH'], + "openimages-calibration-mlperf.json") env['CM_DATASET_CALIBRATION_ANNOTATIONS_FILE_PATH'] = annotations_file_path - return {'return': 0} diff --git a/script/get-dataset-openorca/customize.py b/script/get-dataset-openorca/customize.py index 059c83826d..db4c116ea9 100644 --- a/script/get-dataset-openorca/customize.py +++ b/script/get-dataset-openorca/customize.py @@ -2,19 +2,23 @@ import os import shutil + def preprocess(i): env = i['env'] return {'return': 0} + def postprocess(i): env = i['env'] - if env.get('CM_DATASET_CALIBRATION','') == "no": + if env.get('CM_DATASET_CALIBRATION', '') == "no": env['CM_DATASET_PATH_ROOT'] = env['CM_DATASET_OPENORCA_PATH'] env['CM_DATASET_PATH'] = env['CM_DATASET_OPENORCA_PATH'] - env['CM_DATASET_OPENORCA_PARQUET'] = os.path.join(env['CM_DATASET_OPENORCA_PATH'], '1M-GPT4-Augmented.parquet') + env['CM_DATASET_OPENORCA_PARQUET'] = os.path.join( + env['CM_DATASET_OPENORCA_PATH'], '1M-GPT4-Augmented.parquet') else: - env['CM_CALIBRATION_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'calibration', 'data') + env['CM_CALIBRATION_DATASET_PATH'] = os.path.join( + os.getcwd(), 'install', 'calibration', 'data') return {'return': 0} diff --git a/script/get-dataset-squad-vocab/customize.py b/script/get-dataset-squad-vocab/customize.py index cf869b0094..40d3c5dd56 100644 --- a/script/get-dataset-squad-vocab/customize.py +++ b/script/get-dataset-squad-vocab/customize.py @@ -1,13 +1,14 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] env = i['env'] - return {'return':0} + return {'return': 0} def postprocess(i): @@ -15,4 +16,4 @@ def postprocess(i): env['CM_ML_MODEL_BERT_VOCAB_FILE_WITH_PATH'] = env['CM_DATASET_SQUAD_VOCAB_PATH'] - return {'return':0} + return {'return': 0} diff --git a/script/get-dataset-squad/customize.py b/script/get-dataset-squad/customize.py index fbc01370f1..3af4c85535 100644 --- a/script/get-dataset-squad/customize.py +++ b/script/get-dataset-squad/customize.py @@ -1,20 +1,22 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] env = i['env'] - return {'return':0} + return {'return': 0} def postprocess(i): env = i['env'] - env['CM_DATASET_SQUAD_PATH'] = os.path.dirname(env['CM_DATASET_SQUAD_VAL_PATH']) + env['CM_DATASET_SQUAD_PATH'] = os.path.dirname( + env['CM_DATASET_SQUAD_VAL_PATH']) env['CM_DATASET_PATH'] = os.path.dirname(env['CM_DATASET_SQUAD_VAL_PATH']) - #env['CM_DATASET_SQUAD_VAL_PATH'] = os.path.join(os.getcwd(), env['CM_VAL_FILENAME']) + # env['CM_DATASET_SQUAD_VAL_PATH'] = os.path.join(os.getcwd(), env['CM_VAL_FILENAME']) - return {'return':0} + return {'return': 0} diff --git a/script/get-gcc/customize.py b/script/get-gcc/customize.py index b29f38e13b..9dc79c3ee8 100644 --- a/script/get-gcc/customize.py +++ b/script/get-gcc/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -11,7 +12,8 @@ def preprocess(i): file_name_c = 'gcc.exe' if os_info['platform'] == 'windows' else 'gcc' if env.get('CM_HOST_OS_FLAVOR', '') == 'rhel': - if "12" in env.get('CM_VERSION', '') or "12" in env.get('CM_VERSION_MIN', ''): + if "12" in env.get('CM_VERSION', '') or "12" in env.get( + 'CM_VERSION_MIN', ''): if env.get('CM_TMP_PATH', '') == '': env['CM_TMP_PATH'] = '' env['CM_TMP_PATH'] += "/opt/rh/gcc-toolset-12/root/usr/bin" @@ -20,52 +22,55 @@ def preprocess(i): if 'CM_GCC_BIN_WITH_PATH' not in env: r = i['automation'].find_artifact({'file_name': file_name_c, 'env': env, - 'os_info':os_info, + 'os_info': os_info, 'default_path_env_key': 'PATH', - 'detect_version':True, - 'env_path_key':'CM_GCC_BIN_WITH_PATH', - 'run_script_input':i['run_script_input'], - 'recursion_spaces':recursion_spaces}) - if r['return'] >0 : -# if r['return'] == 16: -# if env.get('CM_TMP_FAIL_IF_NOT_FOUND','').lower() == 'yes': -# return r -# -# print (recursion_spaces+' # {}'.format(r['error'])) -# -# # Attempt to run installer -# r = {'return':0, 'skip':True, 'script':{'tags':'install,gcc,src'}} + 'detect_version': True, + 'env_path_key': 'CM_GCC_BIN_WITH_PATH', + 'run_script_input': i['run_script_input'], + 'recursion_spaces': recursion_spaces}) + if r['return'] > 0: + # if r['return'] == 16: + # if env.get('CM_TMP_FAIL_IF_NOT_FOUND','').lower() == 'yes': + # return r + # + # print (recursion_spaces+' # {}'.format(r['error'])) + # + # # Attempt to run installer + # r = {'return':0, 'skip':True, 'script':{'tags':'install,gcc,src'}} return r - return {'return':0} + return {'return': 0} + def detect_version(i): r = i['automation'].parse_version({'match_text': r' \(.*\)\s*([\d.]+)', 'group_number': 1, - 'env_key':'CM_GCC_VERSION', - 'which_env':i['env']}) - if r['return'] >0: + 'env_key': 'CM_GCC_VERSION', + 'which_env': i['env']}) + if r['return'] > 0: if 'clang' in r['error']: - return {'return':0, 'version':-1} + return {'return': 0, 'version': -1} return r version = r['version'] - print (i['recursion_spaces'] + ' Detected version: {}'.format(version)) + print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + + return {'return': 0, 'version': version} - return {'return':0, 'version':version} def postprocess(i): env = i['env'] r = detect_version(i) - if r['return'] >0: return r + if r['return'] > 0: + return r env['CM_COMPILER_FAMILY'] = 'GCC' version = r['version'] env['CM_COMPILER_VERSION'] = env['CM_GCC_VERSION'] - env['CM_GCC_CACHE_TAGS'] = 'version-'+version - env['CM_COMPILER_CACHE_TAGS'] = 'version-'+version+',family-gcc' + env['CM_GCC_CACHE_TAGS'] = 'version-' + version + env['CM_COMPILER_CACHE_TAGS'] = 'version-' + version + ',family-gcc' found_file_path = env['CM_GCC_BIN_WITH_PATH'] @@ -75,21 +80,21 @@ def postprocess(i): file_name_c = os.path.basename(found_file_path) # G: changed next line to handle cases like gcc-8 - file_name_cpp = file_name_c.replace('gcc','g++') + file_name_cpp = file_name_c.replace('gcc', 'g++') env['FILE_NAME_CPP'] = file_name_cpp - env['CM_GCC_BIN']=file_name_c + env['CM_GCC_BIN'] = file_name_c # General compiler for general program compilation - env['CM_C_COMPILER_BIN']=file_name_c - env['CM_C_COMPILER_FLAG_OUTPUT']='-o ' - env['CM_C_COMPILER_WITH_PATH']=found_file_path - env['CM_C_COMPILER_FLAG_VERSION']='--version' + env['CM_C_COMPILER_BIN'] = file_name_c + env['CM_C_COMPILER_FLAG_OUTPUT'] = '-o ' + env['CM_C_COMPILER_WITH_PATH'] = found_file_path + env['CM_C_COMPILER_FLAG_VERSION'] = '--version' - env['CM_CXX_COMPILER_BIN']=file_name_cpp - env['CM_CXX_COMPILER_WITH_PATH']=os.path.join(found_path, file_name_cpp) - env['CM_CXX_COMPILER_FLAG_OUTPUT']='-o ' - env['CM_CXX_COMPILER_FLAG_VERSION']='--version' + env['CM_CXX_COMPILER_BIN'] = file_name_cpp + env['CM_CXX_COMPILER_WITH_PATH'] = os.path.join(found_path, file_name_cpp) + env['CM_CXX_COMPILER_FLAG_OUTPUT'] = '-o ' + env['CM_CXX_COMPILER_FLAG_VERSION'] = '--version' env['CM_COMPILER_FLAGS_FAST'] = "-O3" env['CM_LINKER_FLAGS_FAST'] = "-O3" @@ -98,5 +103,4 @@ def postprocess(i): env['CM_COMPILER_FLAGS_DEFAULT'] = "-O2" env['CM_LINKER_FLAGS_DEFAULT'] = "-O2" - - return {'return':0, 'version': version} + return {'return': 0, 'version': version} diff --git a/script/get-generic-python-lib/customize.py b/script/get-generic-python-lib/customize.py index ea27ec6603..4b837a79d7 100644 --- a/script/get-generic-python-lib/customize.py +++ b/script/get-generic-python-lib/customize.py @@ -2,6 +2,7 @@ import os import cmind as cm + def preprocess(i): os_info = i['os_info'] @@ -13,21 +14,24 @@ def preprocess(i): package_name = env.get('CM_GENERIC_PYTHON_PACKAGE_NAME', '').strip() if package_name == '': - return automation._available_variations({'meta':meta}) + return automation._available_variations({'meta': meta}) if package_name == "onnxruntime_gpu": # https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#requirements # 20240214: ONNXRuntime 1.17.0 now support CUDA 12 so we remove next check - # TBD: if we have explicit version for ONNX < 17.0.0 and CUDA is >= 12, we should add a check to fail ... - cuda_version = env.get('CM_CUDA_VERSION','').strip() + # TBD: if we have explicit version for ONNX < 17.0.0 and CUDA is >= 12, + # we should add a check to fail ... + cuda_version = env.get('CM_CUDA_VERSION', '').strip() # if cuda_version!='': # cuda_version_split = cuda_version.split('.') # if int(cuda_version_split[0]) >= 12: # # env['CM_INSTALL_ONNXRUNTIME_GPU_FROM_SRC'] = "yes" -# return {'return': 1, 'error':'at this moment, PIP package "onnxruntime_gpu" needs CUDA < 12'} +# return {'return': 1, 'error':'at this moment, PIP package +# "onnxruntime_gpu" needs CUDA < 12'} - extra = env.get('CM_GENERIC_PYTHON_PIP_EXTRA','') - if (pip_version and len(pip_version) > 1 and int(pip_version[0]) >= 23) and ('--break-system-packages' not in extra): + extra = env.get('CM_GENERIC_PYTHON_PIP_EXTRA', '') + if (pip_version and len(pip_version) > 1 and int(pip_version[0]) >= 23) and ( + '--break-system-packages' not in extra): extra += ' --break-system-packages ' env['CM_PYTHON_PIP_COMMON_EXTRA'] = " --break-system-packages" @@ -38,111 +42,135 @@ def preprocess(i): env['CM_PYTHON_PIP_COMMON_EXTRA'] = " --user" if env.get('CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS', '') != '': - r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, 'script_name':'uninstall_deps'}) - if r['return']>0: return r + r = automation.run_native_script( + {'run_script_input': run_script_input, 'env': env, 'script_name': 'uninstall_deps'}) + if r['return'] > 0: + return r prepare_env_key = env.get('CM_GENERIC_PYTHON_PACKAGE_NAME', '') for x in ["-", "[", "]"]: - prepare_env_key = prepare_env_key.replace(x,"_") + prepare_env_key = prepare_env_key.replace(x, "_") env['CM_TMP_PYTHON_PACKAGE_NAME_ENV'] = prepare_env_key.upper() recursion_spaces = i['recursion_spaces'] r = automation.detect_version_using_script({ - 'env': env, - 'run_script_input':i['run_script_input'], - 'recursion_spaces':recursion_spaces}) - - force_install = (env.get('CM_TMP_PYTHON_PACKAGE_FORCE_INSTALL', '') in ['yes', 'true', 'True', True]) - - if r['return'] >0 or force_install: + 'env': env, + 'run_script_input': i['run_script_input'], + 'recursion_spaces': recursion_spaces}) + + force_install = ( + env.get( + 'CM_TMP_PYTHON_PACKAGE_FORCE_INSTALL', + '') in [ + 'yes', + 'true', + 'True', + True]) + + if r['return'] > 0 or force_install: if r['return'] == 16 or force_install: # Clean detected version env if exists otherwise takes detected version # for example, when we reinstall generic python lib package - env_version_key = 'CM_'+env['CM_TMP_PYTHON_PACKAGE_NAME_ENV'].upper()+'_VERSION' - if env.get(env_version_key,'')!='': - del(env[env_version_key]) + env_version_key = 'CM_' + \ + env['CM_TMP_PYTHON_PACKAGE_NAME_ENV'].upper() + '_VERSION' + if env.get(env_version_key, '') != '': + del (env[env_version_key]) # Check if upgrade if force_install: - extra+=' --upgrade --no-deps --force-reinstall' + extra += ' --upgrade --no-deps --force-reinstall' # Check index URL - index_url = env.get('CM_GENERIC_PYTHON_PIP_INDEX_URL','').strip() + index_url = env.get('CM_GENERIC_PYTHON_PIP_INDEX_URL', '').strip() if index_url != '': # Check special cases if '${CM_TORCH_CUDA}' in index_url: - index_url=index_url.replace('${CM_TORCH_CUDA}', env.get('CM_TORCH_CUDA')) + index_url = index_url.replace( + '${CM_TORCH_CUDA}', env.get('CM_TORCH_CUDA')) - extra += ' --index-url '+index_url + extra += ' --index-url ' + index_url # Check extra index URL - extra_index_url = env.get('CM_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL','').strip() + extra_index_url = env.get( + 'CM_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL', '').strip() if extra_index_url != '': # Check special cases if '${CM_TORCH_CUDA}' in extra_index_url: - extra_index_url=extra_index_url.replace('${CM_TORCH_CUDA}', env.get('CM_TORCH_CUDA')) + extra_index_url = extra_index_url.replace( + '${CM_TORCH_CUDA}', env.get('CM_TORCH_CUDA')) - extra += ' --extra-index-url '+extra_index_url + extra += ' --extra-index-url ' + extra_index_url # Check update - if env.get('CM_GENERIC_PYTHON_PIP_UPDATE','') in [True,'true','yes','on']: - extra +=' -U' + if env.get('CM_GENERIC_PYTHON_PIP_UPDATE', '') in [ + True, 'true', 'yes', 'on']: + extra += ' -U' - print ('') - print (recursion_spaces + ' Extra PIP CMD: ' + extra) - print ('') + print('') + print(recursion_spaces + ' Extra PIP CMD: ' + extra) + print('') env['CM_GENERIC_PYTHON_PIP_EXTRA'] = extra - r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, 'script_name':'install'}) + r = automation.run_native_script( + {'run_script_input': run_script_input, 'env': env, 'script_name': 'install'}) + + if r['return'] > 0: + return r - if r['return']>0: return r + return {'return': 0} - return {'return':0} def detect_version(i): env = i['env'] - env_version_key = 'CM_'+env['CM_TMP_PYTHON_PACKAGE_NAME_ENV'].upper()+'_VERSION' + env_version_key = 'CM_' + \ + env['CM_TMP_PYTHON_PACKAGE_NAME_ENV'].upper() + '_VERSION' r = i['automation'].parse_version({'match_text': r'\s*([\d.a-z\-]+)', 'group_number': 1, - 'env_key':env_version_key, - 'which_env':i['env']}) - if r['return'] >0: return r + 'env_key': env_version_key, + 'which_env': i['env']}) + if r['return'] > 0: + return r version = r['version'] current_detected_version = version - if env.get('CM_TMP_SILENT','')!='yes': - print (i['recursion_spaces'] + ' Detected version: {}'.format(version)) + if env.get('CM_TMP_SILENT', '') != 'yes': + print( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) - return {'return':0, 'version':version} + return {'return': 0, 'version': version} def postprocess(i): env = i['env'] - env_version_key = 'CM_'+env['CM_TMP_PYTHON_PACKAGE_NAME_ENV'].upper()+'_VERSION' + env_version_key = 'CM_' + \ + env['CM_TMP_PYTHON_PACKAGE_NAME_ENV'].upper() + '_VERSION' - if env.get(env_version_key,'')!='': + if env.get(env_version_key, '') != '': version = env[env_version_key] else: r = detect_version(i) - if r['return'] >0: return r + if r['return'] > 0: + return r version = r['version'] - env['CM_PYTHONLIB_'+env['CM_TMP_PYTHON_PACKAGE_NAME_ENV']+'_CACHE_TAGS'] = 'version-'+version + env['CM_PYTHONLIB_' + env['CM_TMP_PYTHON_PACKAGE_NAME_ENV'] + + '_CACHE_TAGS'] = 'version-' + version import pkgutil package_name = env.get('CM_GENERIC_PYTHON_PACKAGE_NAME', '').strip() - package=pkgutil.get_loader(package_name) + package = pkgutil.get_loader(package_name) if package: installed_file_path = package.get_filename() env['CM_GET_DEPENDENT_CACHED_PATH'] = installed_file_path @@ -151,4 +179,4 @@ def postprocess(i): if pip_version and len(pip_version) > 1 and int(pip_version[0]) >= 23: env['CM_PYTHON_PIP_COMMON_EXTRA'] = " --break-system-packages" - return {'return':0, 'version': version} + return {'return': 0, 'version': version} diff --git a/script/get-gh-actions-runner/customize.py b/script/get-gh-actions-runner/customize.py index 6c3b91da4b..ea87909aec 100644 --- a/script/get-gh-actions-runner/customize.py +++ b/script/get-gh-actions-runner/customize.py @@ -2,6 +2,7 @@ import os import cmind as cm + def preprocess(i): os_info = i['os_info'] @@ -24,19 +25,21 @@ def preprocess(i): elif cmd == "uninstall": run_cmd = f"cd {env['CM_GH_ACTIONS_RUNNER_CODE_PATH']} && sudo ./svc.sh uninstall" cache_rm_tags = "gh,runner,_install" - r = cm.access({'action': 'rm', 'automation': 'cache', 'tags': cache_rm_tags, 'f': True}) + r = cm.access({'action': 'rm', 'automation': 'cache', + 'tags': cache_rm_tags, 'f': True}) print(r) - if r['return'] != 0 and r['return'] != 16: ## ignore missing ones + if r['return'] != 0 and r['return'] != 16: # ignore missing ones return r elif cmd == "start": run_cmd = f"cd {env['CM_GH_ACTIONS_RUNNER_CODE_PATH']} && sudo ./svc.sh start" env['CM_RUN_CMD'] = run_cmd - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/get-lib-armnn/customize.py b/script/get-lib-armnn/customize.py index 6476cc7894..4fe934f3b6 100644 --- a/script/get-lib-armnn/customize.py +++ b/script/get-lib-armnn/customize.py @@ -1,16 +1,17 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] version = env['CM_LIB_ARMNN_VERSION'] - if env.get('CM_HOST_PLATFORM_FLAVOR','') == 'x86_64': + if env.get('CM_HOST_PLATFORM_FLAVOR', '') == 'x86_64': url = f"https://github.com/ARM-software/armnn/releases/download/{version}/ArmNN-linux-x86_64.tar.gz" - elif env.get('CM_HOST_PLATFORM_FLAVOR','') == 'aarch64': + elif env.get('CM_HOST_PLATFORM_FLAVOR', '') == 'aarch64': url = f"https://github.com/ARM-software/armnn/releases/download/{version}/ArmNN-linux-aarch64.tar.gz" env['CM_LIB_ARMNN_PREBUILT_BINARY_URL'] = url @@ -18,18 +19,19 @@ def preprocess(i): env['CM_GIT_CHECKOUT'] = env['CM_TMP_GIT_BRANCH_NAME'] - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] paths = [ - "+C_INCLUDE_PATH", - "+CPLUS_INCLUDE_PATH", - "+LD_LIBRARY_PATH", - "+DYLD_FALLBACK_LIBRARY_PATH" - ] + "+C_INCLUDE_PATH", + "+CPLUS_INCLUDE_PATH", + "+LD_LIBRARY_PATH", + "+DYLD_FALLBACK_LIBRARY_PATH" + ] for key in paths: env[key] = [] @@ -48,4 +50,4 @@ def postprocess(i): env['+LD_LIBRARY_PATH'].append(lib_path) env['+DYLD_FALLBACK_LIBRARY_PATH'].append(lib_path) - return {'return':0} + return {'return': 0} diff --git a/script/get-lib-dnnl/customize.py b/script/get-lib-dnnl/customize.py index 0d03fd4de0..8b834eb29e 100644 --- a/script/get-lib-dnnl/customize.py +++ b/script/get-lib-dnnl/customize.py @@ -1,28 +1,33 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} + + return {'return': 0} - return {'return':0} def postprocess(i): env = i['env'] env['CM_LIB_DNNL_INSTALL_DIR'] = os.getcwd() - for key in ['+C_INCLUDE_PATH', '+CPLUS_INCLUDE_PATH', '+LD_LIBRARY_PATH', '+DYLD_FALLBACK_LIBRARY_PATH']: -# 20221024: we save and restore env in the main script and can clean env here for determinism -# if key not in env: + for key in ['+C_INCLUDE_PATH', '+CPLUS_INCLUDE_PATH', + '+LD_LIBRARY_PATH', '+DYLD_FALLBACK_LIBRARY_PATH']: + # 20221024: we save and restore env in the main script and can clean env here for determinism + # if key not in env: env[key] = [] - env['+C_INCLUDE_PATH'].append(os.path.join(os.getcwd(), 'install', 'include')) - env['+CPLUS_INCLUDE_PATH'].append(os.path.join(os.getcwd(), 'install', 'include')) + env['+C_INCLUDE_PATH'].append(os.path.join(os.getcwd(), + 'install', 'include')) + env['+CPLUS_INCLUDE_PATH'].append(os.path.join(os.getcwd(), + 'install', 'include')) lib_path = os.path.join(os.getcwd(), 'install', 'lib') env['+LD_LIBRARY_PATH'].append(lib_path) env['+DYLD_FALLBACK_LIBRARY_PATH'].append(lib_path) - return {'return':0} + return {'return': 0} diff --git a/script/get-lib-protobuf/customize.py b/script/get-lib-protobuf/customize.py index c9e641eb44..dc9a423c24 100644 --- a/script/get-lib-protobuf/customize.py +++ b/script/get-lib-protobuf/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -11,10 +12,11 @@ def preprocess(i): automation = i['automation'] - env['CM_GIT_CHECKOUT'] = "v"+env['CM_VERSION'] + env['CM_GIT_CHECKOUT'] = "v" + env['CM_VERSION'] quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): @@ -26,14 +28,21 @@ def postprocess(i): protobuf_install_path = os.path.join(os.getcwd(), "install") env['CM_GOOGLE_PROTOBUF_SRC_PATH'] = env['CM_GIT_REPO_CHECKOUT_PATH'] env['CM_GOOGLE_PROTOBUF_INSTALL_PATH'] = protobuf_install_path - env['+C_INCLUDE_PATH'].append(os.path.join(protobuf_install_path, "include")) - env['+CPLUS_INCLUDE_PATH'].append(os.path.join(protobuf_install_path, "include")) + env['+C_INCLUDE_PATH'].append( + os.path.join( + protobuf_install_path, + "include")) + env['+CPLUS_INCLUDE_PATH'].append( + os.path.join(protobuf_install_path, "include")) if os.path.exists(os.path.join(protobuf_install_path, "lib")): - env['+LD_LIBRARY_PATH'].append(os.path.join(protobuf_install_path, "lib")) + env['+LD_LIBRARY_PATH'].append( + os.path.join(protobuf_install_path, "lib")) elif os.path.exists(os.path.join(protobuf_install_path, "lib64")): - env['+LD_LIBRARY_PATH'].append(os.path.join(protobuf_install_path, "lib64")) + env['+LD_LIBRARY_PATH'].append( + os.path.join(protobuf_install_path, "lib64")) else: - return {'return':1, 'error': f'Protobuf library path not found in {protobuf_install_path}'} + return { + 'return': 1, 'error': f'Protobuf library path not found in {protobuf_install_path}'} - return {'return':0} + return {'return': 0} diff --git a/script/get-lib-qaic-api/customize.py b/script/get-lib-qaic-api/customize.py index 1c95c558cd..4d9b738658 100644 --- a/script/get-lib-qaic-api/customize.py +++ b/script/get-lib-qaic-api/customize.py @@ -1,39 +1,43 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] - #env['CM_GIT_CHECKOUT'] = env['CM_TMP_GIT_BRANCH_NAME'] + # env['CM_GIT_CHECKOUT'] = env['CM_TMP_GIT_BRANCH_NAME'] + + return {'return': 0} - return {'return':0} def postprocess(i): env = i['env'] paths = [ - "+C_INCLUDE_PATH", - "+CPLUS_INCLUDE_PATH", - "+LD_LIBRARY_PATH", - "+DYLD_FALLBACK_LIBRARY_PATH" - ] + "+C_INCLUDE_PATH", + "+CPLUS_INCLUDE_PATH", + "+LD_LIBRARY_PATH", + "+DYLD_FALLBACK_LIBRARY_PATH" + ] for key in paths: env[key] = [] - include_paths = [ env['CM_TMP_CURRENT_SCRIPT_PATH'] ] + include_paths = [env['CM_TMP_CURRENT_SCRIPT_PATH']] for inc_path in include_paths: env['+C_INCLUDE_PATH'].append(inc_path) env['+CPLUS_INCLUDE_PATH'].append(inc_path) version = "master" - env['CM_QAIC_API_SRC_FILE'] = os.path.join(env['CM_TMP_CURRENT_SCRIPT_PATH'], version, "QAicInfApi.cpp") - env['CM_QAIC_API_INC_FILE'] = os.path.join(env['CM_TMP_CURRENT_SCRIPT_PATH'], version, "QAicInfApi.h") + env['CM_QAIC_API_SRC_FILE'] = os.path.join( + env['CM_TMP_CURRENT_SCRIPT_PATH'], version, "QAicInfApi.cpp") + env['CM_QAIC_API_INC_FILE'] = os.path.join( + env['CM_TMP_CURRENT_SCRIPT_PATH'], version, "QAicInfApi.h") - return {'return':0} + return {'return': 0} diff --git a/script/get-microtvm/customize.py b/script/get-microtvm/customize.py index 327760590a..db61f8d735 100644 --- a/script/get-microtvm/customize.py +++ b/script/get-microtvm/customize.py @@ -2,19 +2,21 @@ import os import shutil + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] if 'CM_GIT_DEPTH' not in env: env['CM_GIT_DEPTH'] = '' if 'CM_GIT_RECURSE_SUBMODULES' not in env: env['CM_GIT_RECURSE_SUBMODULES'] = '' - return {'return':0} + return {'return': 0} + def postprocess(i): @@ -23,4 +25,4 @@ def postprocess(i): env['CM_MICROTVM_SOURCE'] = os.path.join(os.getcwd(), 'microtvm') - return {'return':0} + return {'return': 0} diff --git a/script/get-ml-model-gptj/customize.py b/script/get-ml-model-gptj/customize.py index 639efcde87..90343764df 100644 --- a/script/get-ml-model-gptj/customize.py +++ b/script/get-ml-model-gptj/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -8,10 +9,17 @@ def preprocess(i): if env.get('CM_GPTJ_INTEL_MODEL', '') == 'yes': i['run_script_input']['script_name'] = 'run-intel' - harness_root = os.path.join(env['CM_MLPERF_INFERENCE_RESULTS_PATH'], 'closed', 'Intel', 'code', 'gptj-99', 'pytorch-cpu') + harness_root = os.path.join( + env['CM_MLPERF_INFERENCE_RESULTS_PATH'], + 'closed', + 'Intel', + 'code', + 'gptj-99', + 'pytorch-cpu') print(f"Harness Root: {harness_root}") env['CM_HARNESS_CODE_ROOT'] = harness_root - env['CM_CALIBRATION_CODE_ROOT'] = os.path.join(env['CM_MLPERF_INFERENCE_RESULTS_PATH'], 'closed', 'Intel', 'calibration') + env['CM_CALIBRATION_CODE_ROOT'] = os.path.join( + env['CM_MLPERF_INFERENCE_RESULTS_PATH'], 'closed', 'Intel', 'calibration') env['CHECKPOINT_DIR'] = env['GPTJ_CHECKPOINT_PATH'] @@ -24,14 +32,18 @@ def preprocess(i): elif env.get('CM_TMP_ML_MODEL_PROVIDER', '') == 'nvidia': i['run_script_input']['script_name'] = 'run-nvidia' - if str(env.get('CM_DOCKER_DETACHED_MODE','')).lower() in ['yes', 'true', "1"]: + if str(env.get('CM_DOCKER_DETACHED_MODE', '') + ).lower() in ['yes', 'true', "1"]: env['DOCKER_RUN_OPTS'] = "--rm --ipc=host --ulimit memlock=-1 --ulimit stack=67108864" - gpu_arch = int(float(env['CM_CUDA_DEVICE_PROP_GPU_COMPUTE_CAPABILITY']) * 10) + gpu_arch = int( + float( + env['CM_CUDA_DEVICE_PROP_GPU_COMPUTE_CAPABILITY']) * + 10) env['CM_GPU_ARCH'] = gpu_arch env['CM_TMP_REQUIRE_DOWNLOAD'] = 'no' else: - is_saxml = env.get('CM_TMP_MODEL_SAXML','') + is_saxml = env.get('CM_TMP_MODEL_SAXML', '') if is_saxml == "fp32": i['run_script_input']['script_name'] = 'run-saxml' elif is_saxml == "int8": @@ -42,35 +54,46 @@ def preprocess(i): if path == '' or not os.path.exists(path): env['CM_TMP_REQUIRE_DOWNLOAD'] = 'yes' - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - if os.path.exists(os.path.join(env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final")): - env['GPTJ_CHECKPOINT_PATH'] = os.path.join(env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final") + if os.path.exists(os.path.join( + env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final")): + env['GPTJ_CHECKPOINT_PATH'] = os.path.join( + env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final") - is_saxml = env.get('CM_TMP_MODEL_SAXML','') + is_saxml = env.get('CM_TMP_MODEL_SAXML', '') if is_saxml == "fp32": if os.path.exists("pax_gptj_checkpoint"): - env['GPTJ_SAXML_CHECKPOINT_PATH'] = os.path.join(os.getcwd(), "pax_gptj_checkpoint") + env['GPTJ_SAXML_CHECKPOINT_PATH'] = os.path.join( + os.getcwd(), "pax_gptj_checkpoint") env['CM_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_SAXML_CHECKPOINT_PATH'] else: return {'return': 1, 'error': 'pax_gptj_checkpoint generation failed'} elif is_saxml == "int8": if os.path.exists("int8_ckpt"): - env['GPTJ_SAXML_INT8_CHECKPOINT_PATH'] = os.path.join(os.getcwd(), "int8_ckpt") + env['GPTJ_SAXML_INT8_CHECKPOINT_PATH'] = os.path.join( + os.getcwd(), "int8_ckpt") env['CM_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_SAXML_INT8_CHECKPOINT_PATH'] else: return {'return': 1, 'error': 'pax_gptj_checkpoint generation failed'} elif env.get('CM_TMP_ML_MODEL_PROVIDER', '') == 'nvidia': - env['CM_ML_MODEL_FILE_WITH_PATH'] = os.path.join(env['CM_NVIDIA_MLPERF_SCRATCH_PATH'], 'models', 'GPTJ-6B', 'fp8-quantized-ammo', 'GPTJ-FP8-quantized') + env['CM_ML_MODEL_FILE_WITH_PATH'] = os.path.join( + env['CM_NVIDIA_MLPERF_SCRATCH_PATH'], + 'models', + 'GPTJ-6B', + 'fp8-quantized-ammo', + 'GPTJ-FP8-quantized') else: env['CM_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_CHECKPOINT_PATH'] - env['CM_ML_MODEL_FILE'] = os.path.basename(env['CM_ML_MODEL_FILE_WITH_PATH']) + env['CM_ML_MODEL_FILE'] = os.path.basename( + env['CM_ML_MODEL_FILE_WITH_PATH']) env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_ML_MODEL_FILE_WITH_PATH'] - return {'return':0} + return {'return': 0} diff --git a/script/get-ml-model-huggingface-zoo/customize.py b/script/get-ml-model-huggingface-zoo/customize.py index 8770e5bcb4..cd0b5a5c07 100644 --- a/script/get-ml-model-huggingface-zoo/customize.py +++ b/script/get-ml-model-huggingface-zoo/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -18,13 +19,15 @@ def preprocess(i): path = os.getcwd() if env.get('CM_GIT_CLONE_REPO', '') != 'yes': - run_cmd = env.get('CM_PYTHON_BIN_WITH_PATH') + " " + os.path.join(script_path, 'download_model.py') + run_cmd = env.get('CM_PYTHON_BIN_WITH_PATH') + " " + \ + os.path.join(script_path, 'download_model.py') else: run_cmd = '' env['CM_RUN_CMD'] = run_cmd - return {'return':0} + return {'return': 0} + def postprocess(i): @@ -32,19 +35,19 @@ def postprocess(i): env_key = env.get('CM_MODEL_ZOO_ENV_KEY', '') - path_file = env.get('CM_ML_MODEL_FILE_WITH_PATH','') - if path_file!='': + path_file = env.get('CM_ML_MODEL_FILE_WITH_PATH', '') + if path_file != '': path_dir = os.path.dirname(path_file) env['CM_ML_MODEL_PATH'] = path_dir - if env_key!='': - env['CM_ML_MODEL_'+env_key+'_PATH'] = path_dir + if env_key != '': + env['CM_ML_MODEL_' + env_key + '_PATH'] = path_dir else: path_dir = env['CM_ML_MODEL_PATH'] - if env_key!='': - env['CM_ML_MODEL_'+env_key+'_FILE_WITH_PATH'] = path_dir + if env_key != '': + env['CM_ML_MODEL_' + env_key + '_FILE_WITH_PATH'] = path_dir - return {'return':0} + return {'return': 0} diff --git a/script/get-ml-model-llama2/customize.py b/script/get-ml-model-llama2/customize.py index a65e47f828..1f59493125 100644 --- a/script/get-ml-model-llama2/customize.py +++ b/script/get-ml-model-llama2/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -8,7 +9,10 @@ def preprocess(i): if env.get('CM_TMP_ML_MODEL_PROVIDER', '') == 'nvidia': i['run_script_input']['script_name'] = 'run-nvidia' - gpu_arch = int(float(env['CM_CUDA_DEVICE_PROP_GPU_COMPUTE_CAPABILITY']) * 10) + gpu_arch = int( + float( + env['CM_CUDA_DEVICE_PROP_GPU_COMPUTE_CAPABILITY']) * + 10) env['CM_GPU_ARCH'] = gpu_arch env['CM_TMP_REQUIRE_DOWNLOAD'] = 'no' else: @@ -17,24 +21,26 @@ def preprocess(i): if env.get('CM_TMP_ML_MODEL_PROVIDER', '') == 'amd': env['CM_TMP_REQUIRE_DOWNLOAD'] = 'no' i['run_script_input']['script_name'] = 'run-amd' - env['AMD_CODE_DIR'] = os.path.join(env['CM_MLPERF_INFERENCE_RESULTS_PATH'], 'closed', 'AMD', 'code') + env['AMD_CODE_DIR'] = os.path.join( + env['CM_MLPERF_INFERENCE_RESULTS_PATH'], 'closed', 'AMD', 'code') env['CM_LLAMA2_FINAL_SAFE_TENSORS_ROOT'] = os.getcwd() - env['CM_LLAMA2_FINAL_SAFE_TENSORS_PATH'] = os.path.join(env['CM_LLAMA2_FINAL_SAFE_TENSORS_ROOT'], "llama.safetensors") + env['CM_LLAMA2_FINAL_SAFE_TENSORS_PATH'] = os.path.join( + env['CM_LLAMA2_FINAL_SAFE_TENSORS_ROOT'], "llama.safetensors") else: if path == '' or not os.path.exists(path): env['CM_TMP_REQUIRE_DOWNLOAD'] = 'yes' + return {'return': 0} - return {'return':0} def postprocess(i): env = i['env'] - if env.get('LLAMA2_CHECKPOINT_PATH', '' ) == '': + if env.get('LLAMA2_CHECKPOINT_PATH', '') == '': env['LLAMA2_CHECKPOINT_PATH'] = env['CM_ML_MODEL_PATH'] else: env['CM_ML_MODEL_PATH'] = env['LLAMA2_CHECKPOINT_PATH'] env['CM_ML_MODEL_LLAMA2_FILE_WITH_PATH'] = env['LLAMA2_CHECKPOINT_PATH'] env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_ML_MODEL_PATH'] - return {'return':0} + return {'return': 0} diff --git a/script/get-ml-model-mixtral/customize.py b/script/get-ml-model-mixtral/customize.py index 957a87b047..a210368833 100644 --- a/script/get-ml-model-mixtral/customize.py +++ b/script/get-ml-model-mixtral/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -11,7 +12,8 @@ def preprocess(i): if path == '' or not os.path.exists(path): env['CM_TMP_REQUIRE_DOWNLOAD'] = 'yes' - return {'return':0} + return {'return': 0} + def postprocess(i): @@ -23,4 +25,4 @@ def postprocess(i): env['CM_ML_MODEL_PATH'] = env['MIXTRAL_CHECKPOINT_PATH'] env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_ML_MODEL_PATH'] - return {'return':0} + return {'return': 0} diff --git a/script/get-ml-model-neuralmagic-zoo/download_sparse.py b/script/get-ml-model-neuralmagic-zoo/download_sparse.py index 1da36774bd..b2c9de6075 100644 --- a/script/get-ml-model-neuralmagic-zoo/download_sparse.py +++ b/script/get-ml-model-neuralmagic-zoo/download_sparse.py @@ -1,7 +1,7 @@ from sparsezoo import Model import os -model_stub= os.environ.get('CM_MODEL_ZOO_STUB', '') +model_stub = os.environ.get('CM_MODEL_ZOO_STUB', '') print(f"Downloading model {model_stub}") stub = f"{model_stub}" model = Model(stub) diff --git a/script/get-ml-model-resnet50/customize.py b/script/get-ml-model-resnet50/customize.py index 4f30e94181..3aced77567 100644 --- a/script/get-ml-model-resnet50/customize.py +++ b/script/get-ml-model-resnet50/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -10,7 +11,8 @@ def preprocess(i): if env.get('CM_ML_MODEL_TF_FIX_INPUT_SHAPE', '') == "yes": i['run_script_input']['script_name'] = "run-fix-input" - return {'return':0} + return {'return': 0} + def postprocess(i): @@ -18,11 +20,14 @@ def postprocess(i): if env.get('CM_ML_MODEL_TF_FIX_INPUT_SHAPE', '') == "yes": env['CM_ML_MODEL_STARTING_FILE_PATH'] = env['CM_ML_MODEL_FILE_WITH_PATH'] - env['CM_ML_MODEL_FILE_WITH_PATH'] = os.path.join(os.getcwd(), "resnet50_v1.pb") + env['CM_ML_MODEL_FILE_WITH_PATH'] = os.path.join( + os.getcwd(), "resnet50_v1.pb") - env['CM_ML_MODEL_FILE'] = os.path.basename(env['CM_ML_MODEL_FILE_WITH_PATH']) + env['CM_ML_MODEL_FILE'] = os.path.basename( + env['CM_ML_MODEL_FILE_WITH_PATH']) env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_ML_MODEL_FILE_WITH_PATH'] - env['CM_DOWNLOAD_PATH'] = os.path.dirname(env['CM_ML_MODEL_FILE_WITH_PATH']) + env['CM_DOWNLOAD_PATH'] = os.path.dirname( + env['CM_ML_MODEL_FILE_WITH_PATH']) - return {'return':0} + return {'return': 0} diff --git a/script/get-ml-model-retinanet-nvidia/customize.py b/script/get-ml-model-retinanet-nvidia/customize.py index 6da132316e..52fce9129a 100644 --- a/script/get-ml-model-retinanet-nvidia/customize.py +++ b/script/get-ml-model-retinanet-nvidia/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -8,16 +9,30 @@ def preprocess(i): env = i['env'] if '+PYTHONPATH' not in env: env['+PYTHONPATH'] = [] - env['+PYTHONPATH'].append(os.path.join(env['CM_MLPERF_TRAINING_SOURCE'], "single_stage_detector", "ssd")) - env['CM_ML_MODEL_DYN_BATCHSIZE_PATH'] = os.path.join(os.getcwd(), "retinanet_resnext50_32x4d_fpn.opset11.dyn_bs.800x800.onnx") + env['+PYTHONPATH'].append( + os.path.join( + env['CM_MLPERF_TRAINING_SOURCE'], + "single_stage_detector", + "ssd")) + env['CM_ML_MODEL_DYN_BATCHSIZE_PATH'] = os.path.join( + os.getcwd(), "retinanet_resnext50_32x4d_fpn.opset11.dyn_bs.800x800.onnx") if "CM_NVIDIA_EFFICIENT_NMS" in env: - env['CM_NVIDIA_MODEL_PATCHED_PATH'] = os.path.join(os.getcwd(), "fpn_efficientnms_concatall.onnx") - env['CM_ML_MODEL_ANCHOR_PATH'] = os.path.join(env['CM_MLPERF_INFERENCE_NVIDIA_CODE_PATH'], "code", "retinanet", "tensorrt", "onnx_generator", "retinanet_anchor_xywh_1x1.npy") - return {'return':0} + env['CM_NVIDIA_MODEL_PATCHED_PATH'] = os.path.join( + os.getcwd(), "fpn_efficientnms_concatall.onnx") + env['CM_ML_MODEL_ANCHOR_PATH'] = os.path.join( + env['CM_MLPERF_INFERENCE_NVIDIA_CODE_PATH'], + "code", + "retinanet", + "tensorrt", + "onnx_generator", + "retinanet_anchor_xywh_1x1.npy") + return {'return': 0} + def postprocess(i): env = i['env'] - env['CM_NVIDIA_RETINANET_EFFICIENT_NMS_CONCAT_MODEL_WITH_PATH'] = os.path.join(os.getcwd(), "test_fpn_efficientnms_concatall.onnx") + env['CM_NVIDIA_RETINANET_EFFICIENT_NMS_CONCAT_MODEL_WITH_PATH'] = os.path.join( + os.getcwd(), "test_fpn_efficientnms_concatall.onnx") if "CM_NVIDIA_EFFICIENT_NMS" in env: env['CM_NVIDIA_RETINANET_EFFICIENT_NMS_CONCAT_MODEL_WITH_PATH'] = env['CM_NVIDIA_MODEL_PATCHED_PATH'] - return {'return':0} + return {'return': 0} diff --git a/script/get-ml-model-stable-diffusion/customize.py b/script/get-ml-model-stable-diffusion/customize.py index 0f313e49ae..cdaf6b90c2 100644 --- a/script/get-ml-model-stable-diffusion/customize.py +++ b/script/get-ml-model-stable-diffusion/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -11,7 +12,8 @@ def preprocess(i): if path == '' or not os.path.exists(path): env['CM_TMP_REQUIRE_DOWNLOAD'] = 'yes' - return {'return':0} + return {'return': 0} + def postprocess(i): @@ -24,4 +26,4 @@ def postprocess(i): env['CM_GET_DEPENDENT_CACHED_PATH'] = env['SDXL_CHECKPOINT_PATH'] - return {'return':0} + return {'return': 0} diff --git a/script/get-ml-model-using-imagenet-from-model-zoo/customize.py b/script/get-ml-model-using-imagenet-from-model-zoo/customize.py index 4fba39521b..884be01707 100644 --- a/script/get-ml-model-using-imagenet-from-model-zoo/customize.py +++ b/script/get-ml-model-using-imagenet-from-model-zoo/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -11,7 +12,8 @@ def preprocess(i): cm = automation.cmind - return {'return':0} + return {'return': 0} + def postprocess(i): diff --git a/script/get-mlperf-inference-intel-scratch-space/customize.py b/script/get-mlperf-inference-intel-scratch-space/customize.py index 37d9f4a5ed..b88245660a 100644 --- a/script/get-mlperf-inference-intel-scratch-space/customize.py +++ b/script/get-mlperf-inference-intel-scratch-space/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -16,12 +17,13 @@ def preprocess(i): if env.get('CM_INTEL_MLPERF_SCRATCH_PATH', '') == '': env['CM_INTEL_MLPERF_SCRATCH_PATH'] = os.getcwd() - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_INTEL_MLPERF_SCRATCH_PATH'] + env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_INTEL_MLPERF_SCRATCH_PATH'] - return {'return':0} + return {'return': 0} diff --git a/script/get-mlperf-inference-loadgen/customize.py b/script/get-mlperf-inference-loadgen/customize.py index 077a6fae2e..524521e42a 100644 --- a/script/get-mlperf-inference-loadgen/customize.py +++ b/script/get-mlperf-inference-loadgen/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -9,7 +10,8 @@ def preprocess(i): if env.get('CM_TMP_MLPERF_INFERENCE_LOADGEN_INSTALL_FROM_PIP', '') == 'yes': i['run_script_input']['script_name'] = "donotrun" - return {'return':0} + return {'return': 0} + def postprocess(i): @@ -17,12 +19,12 @@ def postprocess(i): env = i['env'] if env.get('CM_TMP_MLPERF_INFERENCE_LOADGEN_INSTALL_FROM_PIP', '') == 'yes': - return {'return':0} - + return {'return': 0} - for key in ['+PYTHONPATH', '+C_INCLUDE_PATH', '+CPLUS_INCLUDE_PATH', '+LD_LIBRARY_PATH', '+DYLD_FALLBACK_LIBRARY_PATH']: -# 20221024: we save and restore env in the main script and can clean env here for determinism -# if key not in env: + for key in ['+PYTHONPATH', '+C_INCLUDE_PATH', '+CPLUS_INCLUDE_PATH', + '+LD_LIBRARY_PATH', '+DYLD_FALLBACK_LIBRARY_PATH']: + # 20221024: we save and restore env in the main script and can clean env here for determinism + # if key not in env: env[key] = [] # On Windows installs directly into Python distro for simplicity @@ -52,4 +54,4 @@ def postprocess(i): env['+PYTHONPATH'].append(python_path) env['CM_MLPERF_INFERENCE_LOADGEN_PYTHON_PATH'] = python_path - return {'return':0} + return {'return': 0} diff --git a/script/get-mlperf-inference-nvidia-common-code/customize.py b/script/get-mlperf-inference-nvidia-common-code/customize.py index 57e0ea43f4..4edd4cbde6 100644 --- a/script/get-mlperf-inference-nvidia-common-code/customize.py +++ b/script/get-mlperf-inference-nvidia-common-code/customize.py @@ -1,19 +1,21 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] env = i['env'] - return {'return':0} + return {'return': 0} def postprocess(i): env = i['env'] - env['CM_MLPERF_INFERENCE_NVIDIA_CODE_PATH'] = os.path.join(env['CM_MLPERF_INFERENCE_RESULTS_PATH'], "closed", "NVIDIA") - env['+PYTHONPATH'] = [ env['CM_MLPERF_INFERENCE_NVIDIA_CODE_PATH'] ] + env['CM_MLPERF_INFERENCE_NVIDIA_CODE_PATH'] = os.path.join( + env['CM_MLPERF_INFERENCE_RESULTS_PATH'], "closed", "NVIDIA") + env['+PYTHONPATH'] = [env['CM_MLPERF_INFERENCE_NVIDIA_CODE_PATH']] - return {'return':0} + return {'return': 0} diff --git a/script/get-mlperf-inference-nvidia-scratch-space/customize.py b/script/get-mlperf-inference-nvidia-scratch-space/customize.py index 1bfa6c9580..5335363893 100644 --- a/script/get-mlperf-inference-nvidia-scratch-space/customize.py +++ b/script/get-mlperf-inference-nvidia-scratch-space/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -14,18 +15,19 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') if env.get('CM_NVIDIA_MLPERF_SCRATCH_PATH', '') == '': - if env.get('MLPERF_SCRATCH_PATH','') != '': + if env.get('MLPERF_SCRATCH_PATH', '') != '': env['CM_NVIDIA_MLPERF_SCRATCH_PATH'] = env['MLPERF_SCRATCH_PATH'] else: env['CM_NVIDIA_MLPERF_SCRATCH_PATH'] = os.getcwd() - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] env['MLPERF_SCRATCH_PATH'] = env['CM_NVIDIA_MLPERF_SCRATCH_PATH'] - env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_NVIDIA_MLPERF_SCRATCH_PATH'] + env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_NVIDIA_MLPERF_SCRATCH_PATH'] - return {'return':0} + return {'return': 0} diff --git a/script/get-mlperf-inference-results-dir/customize.py b/script/get-mlperf-inference-results-dir/customize.py index 8f013816a1..f1beabcaa9 100644 --- a/script/get-mlperf-inference-results-dir/customize.py +++ b/script/get-mlperf-inference-results-dir/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,15 +14,16 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - if env.get('CM_MLPERF_INFERENCE_RESULTS_DIR','') == '': + if env.get('CM_MLPERF_INFERENCE_RESULTS_DIR', '') == '': env['CM_MLPERF_INFERENCE_RESULTS_DIR'] = os.getcwd() - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_MLPERF_INFERENCE_RESULTS_DIR'] + env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_MLPERF_INFERENCE_RESULTS_DIR'] - return {'return':0} + return {'return': 0} diff --git a/script/get-mlperf-inference-results/customize.py b/script/get-mlperf-inference-results/customize.py index 747d99e52d..36f71224de 100644 --- a/script/get-mlperf-inference-results/customize.py +++ b/script/get-mlperf-inference-results/customize.py @@ -2,12 +2,13 @@ import os import shutil + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] meta = i['meta'] @@ -16,7 +17,8 @@ def preprocess(i): env['CM_GIT_URL'] = "https://github.com/GATEOverflow/nvidia-inference-code.git" if 'GITHUB_REPO_OWNER' in env and '<<>>' in env['CM_GIT_URL']: - env['CM_GIT_URL'] = env['CM_GIT_URL'].replace('<<>>', env['GITHUB_REPO_OWNER']) + env['CM_GIT_URL'] = env['CM_GIT_URL'].replace( + '<<>>', env['GITHUB_REPO_OWNER']) if 'CM_GIT_DEPTH' not in env: env['CM_GIT_DEPTH'] = '' @@ -24,13 +26,13 @@ def preprocess(i): if 'CM_GIT_RECURSE_SUBMODULES' not in env: env['CM_GIT_RECURSE_SUBMODULES'] = '' - need_version = env.get('CM_VERSION','') + need_version = env.get('CM_VERSION', '') versions = meta['versions'] - if need_version!='' and not need_version in versions: + if need_version != '' and not need_version in versions: env['CM_GIT_CHECKOUT'] = need_version - return {'return':0} + return {'return': 0} def postprocess(i): @@ -39,8 +41,8 @@ def postprocess(i): state = i['state'] if env.get('CM_GIT_REPO_CURRENT_HASH', '') != '': - env['CM_VERSION'] += "-git-"+env['CM_GIT_REPO_CURRENT_HASH'] + env['CM_VERSION'] += "-git-" + env['CM_GIT_REPO_CURRENT_HASH'] # env['CM_MLPERF_INFERENCE_RESULTS_PATH'] = os.path.join(os.getcwd(), "inference_results_"+env['CM_MLPERF_INFERENCE_RESULTS_VERSION_NAME']) - return {'return':0} + return {'return': 0} diff --git a/script/get-mlperf-inference-submission-dir/customize.py b/script/get-mlperf-inference-submission-dir/customize.py index 92fb3735ce..f0f95f62fd 100644 --- a/script/get-mlperf-inference-submission-dir/customize.py +++ b/script/get-mlperf-inference-submission-dir/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,17 +14,19 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - if env.get('CM_MLPERF_INFERENCE_SUBMISSION_DIR','') == '': + if env.get('CM_MLPERF_INFERENCE_SUBMISSION_DIR', '') == '': if not os.path.exists("mlperf-inference-submission"): os.mkdir("mlperf-inference-submission") - env['CM_MLPERF_INFERENCE_SUBMISSION_DIR'] = os.path.join(os.getcwd(), "mlperf-inference-submission") + env['CM_MLPERF_INFERENCE_SUBMISSION_DIR'] = os.path.join( + os.getcwd(), "mlperf-inference-submission") + + return {'return': 0} - return {'return':0} def postprocess(i): env = i['env'] - env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_MLPERF_INFERENCE_SUBMISSION_DIR'] + env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_MLPERF_INFERENCE_SUBMISSION_DIR'] - return {'return':0} + return {'return': 0} diff --git a/script/get-mlperf-inference-sut-configs/customize.py b/script/get-mlperf-inference-sut-configs/customize.py index 27461d7b24..8adbdc3b8c 100644 --- a/script/get-mlperf-inference-sut-configs/customize.py +++ b/script/get-mlperf-inference-sut-configs/customize.py @@ -3,19 +3,21 @@ import yaml import shutil + def postprocess(i): env = i['env'] state = i['state'] if env.get('CM_HW_NAME', '') == '': - host_name = env.get('CM_HOST_SYSTEM_NAME', 'default').replace("-", "_") + host_name = env.get('CM_HOST_SYSTEM_NAME', 'default').replace("-", "_") env['CM_HW_NAME'] = host_name device = env.get('CM_MLPERF_DEVICE', 'cpu') backend = env.get('CM_MLPERF_BACKEND', 'default') if env.get('CM_MLPERF_BACKEND_VERSION', '') != '': - backend_version = "v" + env.get('CM_MLPERF_BACKEND_VERSION') if not env.get('CM_MLPERF_BACKEND_VERSION').startswith("v") else env.get('CM_MLPERF_BACKEND_VERSION') + backend_version = "v" + env.get('CM_MLPERF_BACKEND_VERSION') if not env.get( + 'CM_MLPERF_BACKEND_VERSION').startswith("v") else env.get('CM_MLPERF_BACKEND_VERSION') else: backend_version = 'vdefault' @@ -24,37 +26,94 @@ def postprocess(i): if 'CM_SUT_CONFIG_PATH' not in state: state['CM_SUT_CONFIG_PATH'] = {} - implementation_string = env['CM_MLPERF_SUT_NAME_IMPLEMENTATION_PREFIX'] if env.get('CM_MLPERF_SUT_NAME_IMPLEMENTATION_PREFIX', '') != '' else env.get('CM_MLPERF_IMPLEMENTATION', 'default') + implementation_string = env['CM_MLPERF_SUT_NAME_IMPLEMENTATION_PREFIX'] if env.get( + 'CM_MLPERF_SUT_NAME_IMPLEMENTATION_PREFIX', '') != '' else env.get( + 'CM_MLPERF_IMPLEMENTATION', 'default') run_config = [] - for i in range(1,6): + for i in range(1, 6): if env.get(f'CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX{i}', '') != '': - run_config.append(env.get(f'CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX{i}')) + run_config.append( + env.get(f'CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX{i}')) - run_config_string = "_".join(run_config) if run_config else 'default_config' + run_config_string = "_".join( + run_config) if run_config else 'default_config' env['CM_MLPERF_INFERENCE_SUT_RUN_CONFIG'] = run_config_string if env.get('CM_SUT_NAME', '') == '': - env['CM_SUT_NAME'] = env['CM_HW_NAME'] + "-" + implementation_string + "-" + device + "-" + backend + "-" + backend_version + "-" + run_config_string + env['CM_SUT_NAME'] = env['CM_HW_NAME'] + "-" + implementation_string + "-" + \ + device + "-" + backend + "-" + backend_version + "-" + run_config_string - if env.get('CM_SUT_CONFIGS_PATH','') != '': + if env.get('CM_SUT_CONFIGS_PATH', '') != '': path = env['CM_SUT_CONFIGS_PATH'] elif env.get('CM_SUT_USE_EXTERNAL_CONFIG_REPO', '') == "yes": path = env.get('CM_GIT_CHECKOUT_PATH') else: path = os.path.join(os.getcwd(), "configs") - config_path = os.path.join(path, env['CM_HW_NAME'], implementation_string+"-implementation", device+"-device", backend+"-framework", "framework-version-"+backend_version, run_config_string + "-config.yaml") + config_path = os.path.join( + path, + env['CM_HW_NAME'], + implementation_string + + "-implementation", + device + + "-device", + backend + + "-framework", + "framework-version-" + + backend_version, + run_config_string + + "-config.yaml") if not os.path.exists(config_path): os.makedirs(os.path.dirname(config_path), exist_ok=True) - config_path_default = os.path.join(path, env['CM_HW_NAME'], implementation_string+"-implementation", device+"-device", backend+"-framework", "default-config.yaml") + config_path_default = os.path.join( + path, + env['CM_HW_NAME'], + implementation_string + + "-implementation", + device + + "-device", + backend + + "-framework", + "default-config.yaml") if os.path.exists(config_path_default): shutil.copy(config_path_default, config_path) else: - src_config_full = os.path.join(env['CM_TMP_CURRENT_SCRIPT_PATH'], "configs", env['CM_HW_NAME'], implementation_string+"-implementation", device+"-device", backend+"-framework", "framework-version-"+backend_version, run_config_string + "-config.yaml") - src_config_partial1 = os.path.join(env['CM_TMP_CURRENT_SCRIPT_PATH'], "configs", env['CM_HW_NAME'], implementation_string+"-implementation", device+"-device", backend+"-framework", "framework-version-"+backend_version, "default-config.yaml") - src_config_partial2 = os.path.join(env['CM_TMP_CURRENT_SCRIPT_PATH'], "configs", env['CM_HW_NAME'], implementation_string+"-implementation", device+"-device", backend+"-framework", "framework-version-default", "default-config.yaml") - src_config_partial3 = os.path.join(env['CM_TMP_CURRENT_SCRIPT_PATH'], "configs", env['CM_HW_NAME'], implementation_string+"-implementation", device+"-device", backend+"-framework", "default-config.yaml") + src_config_full = os.path.join( + env['CM_TMP_CURRENT_SCRIPT_PATH'], + "configs", + env['CM_HW_NAME'], + implementation_string + "-implementation", + device + "-device", + backend + "-framework", + "framework-version-" + backend_version, + run_config_string + "-config.yaml") + src_config_partial1 = os.path.join( + env['CM_TMP_CURRENT_SCRIPT_PATH'], + "configs", + env['CM_HW_NAME'], + implementation_string + "-implementation", + device + "-device", + backend + "-framework", + "framework-version-" + backend_version, + "default-config.yaml") + src_config_partial2 = os.path.join( + env['CM_TMP_CURRENT_SCRIPT_PATH'], + "configs", + env['CM_HW_NAME'], + implementation_string + "-implementation", + device + "-device", + backend + "-framework", + "framework-version-default", + "default-config.yaml") + src_config_partial3 = os.path.join( + env['CM_TMP_CURRENT_SCRIPT_PATH'], + "configs", + env['CM_HW_NAME'], + implementation_string + "-implementation", + device + "-device", + backend + "-framework", + "default-config.yaml") if os.path.exists(src_config_full): shutil.copy(src_config_full, config_path) elif os.path.exists(src_config_partial1): @@ -64,14 +123,22 @@ def postprocess(i): elif os.path.exists(src_config_partial3): shutil.copy(src_config_partial3, config_path) else: - print(f"Config file missing for given hw_name: '{env['CM_HW_NAME']}', implementation: '{implementation_string}', device: '{device}, backend: '{backend}', copying from default") - src_config = os.path.join(env['CM_TMP_CURRENT_SCRIPT_PATH'], "configs", "default", "config.yaml") + print( + f"Config file missing for given hw_name: '{env['CM_HW_NAME']}', implementation: '{implementation_string}', device: '{device}, backend: '{backend}', copying from default") + src_config = os.path.join( + env['CM_TMP_CURRENT_SCRIPT_PATH'], + "configs", + "default", + "config.yaml") shutil.copy(src_config, config_path) - os.makedirs(os.path.dirname(config_path_default), exist_ok=True) + os.makedirs( + os.path.dirname(config_path_default), + exist_ok=True) shutil.copy(src_config, config_path_default) - state['CM_SUT_CONFIG'][env['CM_SUT_NAME']] = yaml.load(open(config_path), Loader=yaml.SafeLoader) + state['CM_SUT_CONFIG'][env['CM_SUT_NAME']] = yaml.load( + open(config_path), Loader=yaml.SafeLoader) state['CM_SUT_CONFIG_NAME'] = env['CM_SUT_NAME'] state['CM_SUT_CONFIG_PATH'][env['CM_SUT_NAME']] = config_path - return {'return':0} + return {'return': 0} diff --git a/script/get-mlperf-inference-utils/customize.py b/script/get-mlperf-inference-utils/customize.py index 8c4bbbbbf4..6f7f0a49b8 100644 --- a/script/get-mlperf-inference-utils/customize.py +++ b/script/get-mlperf-inference-utils/customize.py @@ -2,6 +2,7 @@ import os import sys + def preprocess(i): os_info = i['os_info'] @@ -16,17 +17,19 @@ def preprocess(i): utils_path = i['run_script_input']['path'] - env['+PYTHONPATH'] = [ utils_path ] + env['+PYTHONPATH'] = [utils_path] - submission_checker_dir = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "tools", "submission") + submission_checker_dir = os.path.join( + env['CM_MLPERF_INFERENCE_SOURCE'], "tools", "submission") sys.path.append(submission_checker_dir) sys.path.append(utils_path) - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/get-mlperf-logging/customize.py b/script/get-mlperf-logging/customize.py index ac1a2a641e..fea70fc389 100644 --- a/script/get-mlperf-logging/customize.py +++ b/script/get-mlperf-logging/customize.py @@ -2,6 +2,7 @@ import os import shutil + def preprocess(i): os_info = i['os_info'] @@ -11,11 +12,12 @@ def preprocess(i): env['CM_MLPERF_LOGGING_SRC_PATH'] = env['CM_GIT_REPO_CHECKOUT_PATH'] - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - env['+PYTHONPATH'] = [ env['CM_MLPERF_LOGGING_SRC_PATH'] ] + env['+PYTHONPATH'] = [env['CM_MLPERF_LOGGING_SRC_PATH']] - return {'return':0} + return {'return': 0} diff --git a/script/get-mlperf-power-dev/customize.py b/script/get-mlperf-power-dev/customize.py index 50afb3ba49..d091365bb7 100644 --- a/script/get-mlperf-power-dev/customize.py +++ b/script/get-mlperf-power-dev/customize.py @@ -2,11 +2,12 @@ import os import shutil + def preprocess(i): os_info = i['os_info'] - return {'return':0} + return {'return': 0} def postprocess(i): @@ -16,6 +17,6 @@ def postprocess(i): env['CM_VERSION'] = "master" if env.get('CM_GIT_REPO_CURRENT_HASH', '') != '': - env['CM_VERSION'] += "-git-"+env['CM_GIT_REPO_CURRENT_HASH'] + env['CM_VERSION'] += "-git-" + env['CM_GIT_REPO_CURRENT_HASH'] - return {'return':0, 'version': env['CM_VERSION']} + return {'return': 0, 'version': env['CM_VERSION']} diff --git a/script/get-mlperf-tiny-eembc-energy-runner-src/customize.py b/script/get-mlperf-tiny-eembc-energy-runner-src/customize.py index 93a162b980..a2e5243154 100644 --- a/script/get-mlperf-tiny-eembc-energy-runner-src/customize.py +++ b/script/get-mlperf-tiny-eembc-energy-runner-src/customize.py @@ -2,12 +2,14 @@ import os import shutil + def preprocess(i): os_info = i['os_info'] # if os_info['platform'] == 'windows': -# return {'return':1, 'error': 'Windows is not supported in this script yet'} +# return {'return':1, 'error': 'Windows is not supported in this script +# yet'} env = i['env'] meta = i['meta'] @@ -15,7 +17,7 @@ def preprocess(i): if 'CM_GIT_DEPTH' not in env: env['CM_GIT_DEPTH'] = '' - return {'return':0} + return {'return': 0} def postprocess(i): @@ -28,31 +30,37 @@ def postprocess(i): env['CM_EEMBC_ENERGY_RUNNER_SRC_DATASETS'] = datasets_src_path # Get user directory for EEMBC runner path - home_directory = os.path.expanduser( '~' ) + home_directory = os.path.expanduser('~') sessions_path = os.path.join(home_directory, 'eembc', 'runner', 'sessions') - print ('') - print ('Path to EEMBC runner sessions: {}'.format(sessions_path)) + print('') + print('Path to EEMBC runner sessions: {}'.format(sessions_path)) env['CM_EEMBC_ENERGY_RUNNER_SESSIONS'] = sessions_path if not os.path.isdir(sessions_path): os.makedirs(sessions_path) - datasets_path = os.path.join(home_directory, 'eembc', 'runner', 'benchmarks', 'ulp-mlperf', 'datasets') + datasets_path = os.path.join( + home_directory, + 'eembc', + 'runner', + 'benchmarks', + 'ulp-mlperf', + 'datasets') - print ('') - print ('Path to EEMBC runner datasets: {}'.format(datasets_path)) + print('') + print('Path to EEMBC runner datasets: {}'.format(datasets_path)) if not os.path.isdir(datasets_path): os.makedirs(datasets_path) env['CM_EEMBC_ENERGY_RUNNER_DATASETS'] = datasets_path - print ('') - print ('Copying datasets to EEMBC user space ...') + print('') + print('Copying datasets to EEMBC user space ...') shutil.copytree(datasets_src_path, datasets_path, dirs_exist_ok=True) - return {'return':0} + return {'return': 0} diff --git a/script/get-mlperf-tiny-src/customize.py b/script/get-mlperf-tiny-src/customize.py index f361f009f8..f07ee475db 100644 --- a/script/get-mlperf-tiny-src/customize.py +++ b/script/get-mlperf-tiny-src/customize.py @@ -2,12 +2,14 @@ import os import shutil + def preprocess(i): os_info = i['os_info'] # if os_info['platform'] == 'windows': -# return {'return':1, 'error': 'Windows is not supported in this script yet'} +# return {'return':1, 'error': 'Windows is not supported in this script +# yet'} env = i['env'] meta = i['meta'] @@ -15,7 +17,7 @@ def preprocess(i): if 'CM_GIT_DEPTH' not in env: env['CM_GIT_DEPTH'] = '' - return {'return':0} + return {'return': 0} def postprocess(i): @@ -24,18 +26,30 @@ def postprocess(i): state = i['state'] env['CM_MLPERF_TINY_SRC'] = os.path.join(os.getcwd(), 'src') - env['CM_MLPERF_TINY_BENCHMARK'] = os.path.join(os.getcwd(), 'src', 'benchmark') - env['CM_MLPERF_TINY_DATASETS'] = os.path.join(os.getcwd(), 'src', 'benchmark', 'evaluation', 'datasets') - env['CM_MLPERF_TINY_DATASETS_AD'] = os.path.join(os.getcwd(), 'src', 'benchmark', 'evaluation', 'datasets', 'ad01') - env['CM_MLPERF_TINY_DATASETS_IC'] = os.path.join(os.getcwd(), 'src', 'benchmark', 'evaluation', 'datasets', 'ic01') - env['CM_MLPERF_TINY_DATASETS_KWS'] = os.path.join(os.getcwd(), 'src', 'benchmark', 'evaluation', 'datasets', 'kws01') - env['CM_MLPERF_TINY_DATASETS_KWS_OPEN'] = os.path.join(os.getcwd(), 'src', 'benchmark', 'evaluation', 'datasets', 'kws01-open') - env['CM_MLPERF_TINY_DATASETS_VWW'] = os.path.join(os.getcwd(), 'src', 'benchmark', 'evaluation', 'datasets', 'vww01') - env['CM_MLPERF_TINY_TRAINING'] = os.path.join(os.getcwd(), 'src', 'benchmark', 'training') - env['CM_MLPERF_TINY_TRAINING_AD'] = os.path.join(os.getcwd(), 'src', 'benchmark', 'training', 'anomaly_detection') - env['CM_MLPERF_TINY_TRAINING_IC'] = os.path.join(os.getcwd(), 'src', 'benchmark', 'training', 'image_classification') - env['CM_MLPERF_TINY_TRAINING_KWS'] = os.path.join(os.getcwd(), 'src', 'benchmark', 'training', 'keyword_spotting') - env['CM_MLPERF_TINY_TRAINING_VWW'] = os.path.join(os.getcwd(), 'src', 'benchmark', 'training', 'visual_wake_words') + env['CM_MLPERF_TINY_BENCHMARK'] = os.path.join( + os.getcwd(), 'src', 'benchmark') + env['CM_MLPERF_TINY_DATASETS'] = os.path.join( + os.getcwd(), 'src', 'benchmark', 'evaluation', 'datasets') + env['CM_MLPERF_TINY_DATASETS_AD'] = os.path.join( + os.getcwd(), 'src', 'benchmark', 'evaluation', 'datasets', 'ad01') + env['CM_MLPERF_TINY_DATASETS_IC'] = os.path.join( + os.getcwd(), 'src', 'benchmark', 'evaluation', 'datasets', 'ic01') + env['CM_MLPERF_TINY_DATASETS_KWS'] = os.path.join( + os.getcwd(), 'src', 'benchmark', 'evaluation', 'datasets', 'kws01') + env['CM_MLPERF_TINY_DATASETS_KWS_OPEN'] = os.path.join( + os.getcwd(), 'src', 'benchmark', 'evaluation', 'datasets', 'kws01-open') + env['CM_MLPERF_TINY_DATASETS_VWW'] = os.path.join( + os.getcwd(), 'src', 'benchmark', 'evaluation', 'datasets', 'vww01') + env['CM_MLPERF_TINY_TRAINING'] = os.path.join( + os.getcwd(), 'src', 'benchmark', 'training') + env['CM_MLPERF_TINY_TRAINING_AD'] = os.path.join( + os.getcwd(), 'src', 'benchmark', 'training', 'anomaly_detection') + env['CM_MLPERF_TINY_TRAINING_IC'] = os.path.join( + os.getcwd(), 'src', 'benchmark', 'training', 'image_classification') + env['CM_MLPERF_TINY_TRAINING_KWS'] = os.path.join( + os.getcwd(), 'src', 'benchmark', 'training', 'keyword_spotting') + env['CM_MLPERF_TINY_TRAINING_VWW'] = os.path.join( + os.getcwd(), 'src', 'benchmark', 'training', 'visual_wake_words') # 20221024: we save and restore env in the main script and can clean env here for determinism # if '+PYTHONPATH' not in env: env['+PYTHONPATH'] = [] @@ -43,4 +57,4 @@ def postprocess(i): # env['+PYTHONPATH'].append(os.path.join(env['CM_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], 'python')) # env['+PYTHONPATH'].append(os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], 'tools', 'submission')) - return {'return':0} + return {'return': 0} diff --git a/script/get-mlperf-training-nvidia-code/customize.py b/script/get-mlperf-training-nvidia-code/customize.py index a58acfbad3..4a2d1f33d0 100644 --- a/script/get-mlperf-training-nvidia-code/customize.py +++ b/script/get-mlperf-training-nvidia-code/customize.py @@ -1,22 +1,25 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] env = i['env'] - return {'return':0} + return {'return': 0} def postprocess(i): env = i['env'] - env['CM_MLPERF_TRAINING_NVIDIA_CODE_PATH'] = os.path.join(env['CM_MLPERF_TRAINING_RESULTS_PATH'], "NVIDIA") + env['CM_MLPERF_TRAINING_NVIDIA_CODE_PATH'] = os.path.join( + env['CM_MLPERF_TRAINING_RESULTS_PATH'], "NVIDIA") if not os.path.exists(env['CM_MLPERF_TRAINING_NVIDIA_CODE_PATH']): - return {'return': 1, 'error': f'Nvidia code path not found in the repository{env["CM_MLPERF_TRAINING_RESULTS_PATH"]}'} + return { + 'return': 1, 'error': f'Nvidia code path not found in the repository{env["CM_MLPERF_TRAINING_RESULTS_PATH"]}'} env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_MLPERF_TRAINING_NVIDIA_CODE_PATH'] - return {'return':0} + return {'return': 0} diff --git a/script/get-mlperf-training-src/customize.py b/script/get-mlperf-training-src/customize.py index d2de607ed2..1af8336a89 100644 --- a/script/get-mlperf-training-src/customize.py +++ b/script/get-mlperf-training-src/customize.py @@ -2,6 +2,7 @@ import os import shutil + def preprocess(i): env = i['env'] @@ -12,11 +13,12 @@ def preprocess(i): patch_files = env['CM_GIT_PATCH_FILENAMES'].split(",") patch_files_full_paths = [] for patch_file in patch_files: - patch_file_full_path = os.path.join(script_path, "patch", patch_file) + patch_file_full_path = os.path.join( + script_path, "patch", patch_file) patch_files_full_paths.append(patch_file_full_path) env['CM_GIT_PATCH_FILEPATHS'] = ",".join(patch_files_full_paths) - return {'return':0} + return {'return': 0} def postprocess(i): @@ -24,4 +26,4 @@ def postprocess(i): env = i['env'] state = i['state'] - return {'return':0} + return {'return': 0} diff --git a/script/get-openssl/customize.py b/script/get-openssl/customize.py index 9d126fd79e..2ac5ac73b8 100644 --- a/script/get-openssl/customize.py +++ b/script/get-openssl/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,45 +14,49 @@ def preprocess(i): if 'CM_OPENSSL_BIN_WITH_PATH' not in env: r = i['automation'].find_artifact({'file_name': file_name, 'env': env, - 'os_info':os_info, + 'os_info': os_info, 'default_path_env_key': 'PATH', - 'detect_version':True, - 'env_path_key':'CM_OPENSSL_BIN_WITH_PATH', - 'run_script_input':i['run_script_input'], - 'recursion_spaces':i['recursion_spaces']}) - if r['return']>0: + 'detect_version': True, + 'env_path_key': 'CM_OPENSSL_BIN_WITH_PATH', + 'run_script_input': i['run_script_input'], + 'recursion_spaces': i['recursion_spaces']}) + if r['return'] > 0: if r['return'] == 16 and os_info['platform'] != 'windows': env['CM_REQUIRE_INSTALL'] = "yes" return {'return': 0} return r - return {'return':0} + return {'return': 0} + def detect_version(i): r = i['automation'].parse_version({'match_text': r'OpenSSL\s*([\d.]+)', 'group_number': 1, - 'env_key':'CM_OPENSSL_VERSION', - 'which_env':i['env']}) - if r['return'] >0: return r + 'env_key': 'CM_OPENSSL_VERSION', + 'which_env': i['env']}) + if r['return'] > 0: + return r version = r['version'] - print (i['recursion_spaces'] + ' Detected version: {}'.format(version)) - return {'return':0, 'version':version} + print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + return {'return': 0, 'version': version} def postprocess(i): env = i['env'] r = detect_version(i) - if r['return'] >0: return r + if r['return'] > 0: + return r version = r['version'] found_file_path = env['CM_OPENSSL_BIN_WITH_PATH'] found_path = os.path.dirname(found_file_path) env['CM_OPENSSL_INSTALLED_PATH'] = found_path - # Save tags that can be used to specialize further dependencies (such as python packages) - tags = 'version-'+version + # Save tags that can be used to specialize further dependencies (such as + # python packages) + tags = 'version-' + version - return {'return':0, 'version': version} + return {'return': 0, 'version': version} diff --git a/script/get-preprocessed-dataset-criteo/customize.py b/script/get-preprocessed-dataset-criteo/customize.py index 3fc23050d0..bb60894f6d 100644 --- a/script/get-preprocessed-dataset-criteo/customize.py +++ b/script/get-preprocessed-dataset-criteo/customize.py @@ -2,6 +2,7 @@ import os import shutil + def preprocess(i): env = i['env'] @@ -12,24 +13,33 @@ def preprocess(i): Path with preprocessed dataset given as input ''' skip_preprocessing = True - print("Using preprocessed criteo dataset from '" + env['CM_DATASET_PREPROCESSED_PATH'] +"'") + print("Using preprocessed criteo dataset from '" + + env['CM_DATASET_PREPROCESSED_PATH'] + "'") - if not skip_preprocessing and env.get('CM_DATASET_PREPROCESSED_OUTPUT_PATH','') != '': + if not skip_preprocessing and env.get( + 'CM_DATASET_PREPROCESSED_OUTPUT_PATH', '') != '': env['CM_DATASET_PREPROCESSED_PATH'] = os.getcwd() - if not skip_preprocessing and env.get('CM_DATASET_CRITEO_MULTIHOT', '') == 'yes': + if not skip_preprocessing and env.get( + 'CM_DATASET_CRITEO_MULTIHOT', '') == 'yes': i['run_script_input']['script_name'] = "run-multihot" - #${CM_PYTHON_BIN_WITH_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/preprocess.py + # ${CM_PYTHON_BIN_WITH_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/preprocess.py output_dir = env['CM_DATASET_PREPROCESSED_PATH'] dataset_path = env['CM_DATASET_PATH'] tmp_dir = os.path.join(output_dir, "tmp") - run_dir = os.path.join(env['CM_MLPERF_TRAINING_SOURCE'], "recommendation_v2", "torchrec_dlrm", "scripts") + run_dir = os.path.join( + env['CM_MLPERF_TRAINING_SOURCE'], + "recommendation_v2", + "torchrec_dlrm", + "scripts") env['CM_RUN_CMD'] = f'cd {run_dir} && bash ./process_Criteo_1TB_Click_Logs_dataset.sh {dataset_path} {tmp_dir} {output_dir} ' - print("Using MLCommons Training source from '" + env['CM_MLPERF_TRAINING_SOURCE'] +"'") + print("Using MLCommons Training source from '" + + env['CM_MLPERF_TRAINING_SOURCE'] + "'") return {'return': 0} + def postprocess(i): env = i['env'] diff --git a/script/get-preprocessed-dataset-criteo/preprocess.py b/script/get-preprocessed-dataset-criteo/preprocess.py index bd2e5f5430..5a5c429c62 100644 --- a/script/get-preprocessed-dataset-criteo/preprocess.py +++ b/script/get-preprocessed-dataset-criteo/preprocess.py @@ -1,32 +1,35 @@ +import dataset +import criteo import os import sys mlperf_dlrm_path = os.environ['CM_MLPERF_INFERENCE_DLRM_PATH'] python_path = os.path.join(mlperf_dlrm_path, "pytorch", "python") sys.path.insert(0, python_path) -import criteo -import dataset dataset_name = os.environ['CM_DATASET'] dataset_path = os.environ['CM_DATASET_PATH'] dataset_list = os.environ.get('CM_DATASET_IMAGES_LIST', None) -samples_to_aggregate_fix = os.environ.get('CM_DATASET_SAMPLES_TO_AGGREGATE_FIX', None) -samples_to_aggregate_min = os.environ.get('CM_DATASET_SAMPLES_TO_AGGREGATE_MIN', None) -samples_to_aggregate_max = os.environ.get('CM_DATASET_SAMPLES_TO_AGGREGATE_MAX', None) +samples_to_aggregate_fix = os.environ.get( + 'CM_DATASET_SAMPLES_TO_AGGREGATE_FIX', None) +samples_to_aggregate_min = os.environ.get( + 'CM_DATASET_SAMPLES_TO_AGGREGATE_MIN', None) +samples_to_aggregate_max = os.environ.get( + 'CM_DATASET_SAMPLES_TO_AGGREGATE_MAX', None) count = int(os.environ.get('CM_DATASET_SIZE', 0)) or None -max_ind_range = os.environ.get('CM_DATASET_MAX_IND_RANGE',-1) +max_ind_range = os.environ.get('CM_DATASET_MAX_IND_RANGE', -1) threads = os.environ.get('CM_NUM_THREADS', os.cpu_count()) threads = os.environ.get('CM_NUM_PREPROCESS_THREADS', threads) criteo.Criteo(data_path=dataset_path, - name=dataset_name, - pre_process = criteo.pre_process_criteo_dlrm, - use_cache=True, - samples_to_aggregate_fix=samples_to_aggregate_fix, - samples_to_aggregate_min=samples_to_aggregate_min, - samples_to_aggregate_max=samples_to_aggregate_max, - max_ind_range=max_ind_range, - count=count, - mlperf_bin_loader=False, - test_num_workers=threads - ) + name=dataset_name, + pre_process=criteo.pre_process_criteo_dlrm, + use_cache=True, + samples_to_aggregate_fix=samples_to_aggregate_fix, + samples_to_aggregate_min=samples_to_aggregate_min, + samples_to_aggregate_max=samples_to_aggregate_max, + max_ind_range=max_ind_range, + count=count, + mlperf_bin_loader=False, + test_num_workers=threads + ) diff --git a/script/get-preprocessed-dataset-generic/customize.py b/script/get-preprocessed-dataset-generic/customize.py index b882aaf93d..c470734b90 100644 --- a/script/get-preprocessed-dataset-generic/customize.py +++ b/script/get-preprocessed-dataset-generic/customize.py @@ -1,10 +1,11 @@ from cmind import utils import os + def preprocess(i): env = i['env'] path = i['run_script_input']['path'] - env['+PYTHONPATH'] = [ os.path.join(path, "src") ] + env['+PYTHONPATH'] = [os.path.join(path, "src")] return {'return': 0} diff --git a/script/get-preprocessed-dataset-kits19/customize.py b/script/get-preprocessed-dataset-kits19/customize.py index c8a0914d24..3b108f88a8 100644 --- a/script/get-preprocessed-dataset-kits19/customize.py +++ b/script/get-preprocessed-dataset-kits19/customize.py @@ -2,17 +2,25 @@ import os import shutil + def preprocess(i): env = i['env'] - print("Using MLCommons Inference source from '" + env['CM_MLPERF_INFERENCE_SOURCE'] +"'") - preprocess_src = os.path.join(env['CM_MLPERF_INFERENCE_3DUNET_PATH'], 'preprocess.py') - cmd = 'cd '+ env['CM_MLPERF_INFERENCE_3DUNET_PATH'] + ' && ${CM_PYTHON_BIN_WITH_PATH} preprocess.py --raw_data_dir ' + env['CM_DATASET_PATH'] + ' --results_dir ' + os.getcwd() + ' --mode preprocess' + print("Using MLCommons Inference source from '" + + env['CM_MLPERF_INFERENCE_SOURCE'] + "'") + preprocess_src = os.path.join( + env['CM_MLPERF_INFERENCE_3DUNET_PATH'], + 'preprocess.py') + cmd = 'cd ' + env['CM_MLPERF_INFERENCE_3DUNET_PATH'] + \ + ' && ${CM_PYTHON_BIN_WITH_PATH} preprocess.py --raw_data_dir ' + \ + env['CM_DATASET_PATH'] + ' --results_dir ' + \ + os.getcwd() + ' --mode preprocess' env['CM_TMP_CMD'] = cmd return {'return': 0} + def postprocess(i): env = i['env'] if 'CM_DATASET_PREPROCESSED_PATH' not in env: diff --git a/script/get-preprocessed-dataset-librispeech/customize.py b/script/get-preprocessed-dataset-librispeech/customize.py index e5a8a12e2b..7a3f6c73f0 100644 --- a/script/get-preprocessed-dataset-librispeech/customize.py +++ b/script/get-preprocessed-dataset-librispeech/customize.py @@ -2,20 +2,31 @@ import os import shutil + def preprocess(i): env = i['env'] - print("Using MLCommons Inference source from '" + env['CM_MLPERF_INFERENCE_SOURCE'] +"'") - preprocess_src = os.path.join(env['CM_MLPERF_INFERENCE_RNNT_PATH'], 'pytorch', 'utils', 'convert_librispeech.py') - cmd = 'cd '+ env['CM_MLPERF_INFERENCE_3DUNET_PATH'] + ' && ${CM_PYTHON_BIN_WITH_PATH} ' + preprocess_src + ' --input_dir ' + env['CM_DATASET_LIBRISPEECH_PATH'] + ' --dest_dir ' + os.path.join(os.getcwd(), 'dev-clean-wav') + ' --output_json ' + os.path.join(os.getcwd(), 'dev-clean-wav.json') + print("Using MLCommons Inference source from '" + + env['CM_MLPERF_INFERENCE_SOURCE'] + "'") + preprocess_src = os.path.join( + env['CM_MLPERF_INFERENCE_RNNT_PATH'], + 'pytorch', + 'utils', + 'convert_librispeech.py') + cmd = 'cd ' + env['CM_MLPERF_INFERENCE_3DUNET_PATH'] + ' && ${CM_PYTHON_BIN_WITH_PATH} ' + preprocess_src + ' --input_dir ' + env['CM_DATASET_LIBRISPEECH_PATH'] + \ + ' --dest_dir ' + os.path.join(os.getcwd(), 'dev-clean-wav') + \ + ' --output_json ' + os.path.join(os.getcwd(), 'dev-clean-wav.json') env['CM_TMP_CMD'] = cmd return {'return': 0} + def postprocess(i): env = i['env'] - env['CM_DATASET_PREPROCESSED_PATH'] = os.path.join(os.getcwd(), 'dev-clean-wav') - env['CM_DATASET_PREPROCESSED_JSON'] = os.path.join(os.getcwd(), 'dev-clean-wav.json') + env['CM_DATASET_PREPROCESSED_PATH'] = os.path.join( + os.getcwd(), 'dev-clean-wav') + env['CM_DATASET_PREPROCESSED_JSON'] = os.path.join( + os.getcwd(), 'dev-clean-wav.json') return {'return': 0} diff --git a/script/get-preprocessed-dataset-openimages/customize.py b/script/get-preprocessed-dataset-openimages/customize.py index fd2adcb5f6..cec47ecea5 100644 --- a/script/get-preprocessed-dataset-openimages/customize.py +++ b/script/get-preprocessed-dataset-openimages/customize.py @@ -3,6 +3,7 @@ import shutil import glob + def preprocess(i): env = i['env'] @@ -10,8 +11,9 @@ def preprocess(i): if 'CM_DATASET_PREPROCESSED_PATH' not in env: env['CM_DATASET_PREPROCESSED_PATH'] = os.getcwd() - if env.get('CM_DATASET_REFERENCE_PREPROCESSOR',"0") == "1": - print("Using MLCommons Inference source from '" + env['CM_MLPERF_INFERENCE_SOURCE'] +"'") + if env.get('CM_DATASET_REFERENCE_PREPROCESSOR', "0") == "1": + print("Using MLCommons Inference source from '" + + env['CM_MLPERF_INFERENCE_SOURCE'] + "'") if env.get('CM_ML_MODEL_NAME', '') == 'retinanet': if env.get('CM_DATASET_QUANTIZE', '') == '1': @@ -22,20 +24,23 @@ def preprocess(i): return {'return': 0} + def postprocess(i): env = i['env'] if env["CM_DATASET_TYPE"] == "validation": - env['CM_DATASET_ANNOTATIONS_DIR_PATH'] = os.path.join(env['CM_DATASET_PREPROCESSED_PATH'], "annotations") - env['CM_DATASET_ANNOTATIONS_FILE_PATH'] = os.path.join(env['CM_DATASET_ANNOTATIONS_DIR_PATH'], "openimages-mlperf.json") + env['CM_DATASET_ANNOTATIONS_DIR_PATH'] = os.path.join( + env['CM_DATASET_PREPROCESSED_PATH'], "annotations") + env['CM_DATASET_ANNOTATIONS_FILE_PATH'] = os.path.join( + env['CM_DATASET_ANNOTATIONS_DIR_PATH'], "openimages-mlperf.json") # finalize path preprocessed_path = env['CM_DATASET_PREPROCESSED_PATH'] preprocessed_images_list = [] preprocessed_imagenames_list = [] - match_text = "/*."+env.get("CM_DATASET_PREPROCESSED_EXTENSION","*") + match_text = "/*." + env.get("CM_DATASET_PREPROCESSED_EXTENSION", "*") for filename in sorted(glob.glob(preprocessed_path + match_text)): preprocessed_images_list.append(filename) preprocessed_imagenames_list.append(os.path.basename(filename)) @@ -44,7 +49,9 @@ def postprocess(i): with open("preprocessed_filenames.txt", "w") as f: f.write("\n".join(preprocessed_imagenames_list)) - env['CM_DATASET_PREPROCESSED_IMAGES_LIST'] = os.path.join(os.getcwd(), "preprocessed_files.txt") - env['CM_DATASET_PREPROCESSED_IMAGENAMES_LIST'] = os.path.join(os.getcwd(), "preprocessed_filenames.txt") + env['CM_DATASET_PREPROCESSED_IMAGES_LIST'] = os.path.join( + os.getcwd(), "preprocessed_files.txt") + env['CM_DATASET_PREPROCESSED_IMAGENAMES_LIST'] = os.path.join( + os.getcwd(), "preprocessed_filenames.txt") return {'return': 0} diff --git a/script/get-preprocessed-dataset-openimages/preprocess.py b/script/get-preprocessed-dataset-openimages/preprocess.py index b2b05fe1dc..c5af0ff041 100644 --- a/script/get-preprocessed-dataset-openimages/preprocess.py +++ b/script/get-preprocessed-dataset-openimages/preprocess.py @@ -1,3 +1,6 @@ +import shutil +import dataset +import openimages import os import sys import os.path @@ -6,17 +9,13 @@ python_path = os.path.join(mlperf_src_path, "python") sys.path.insert(0, python_path) -import openimages -import dataset -import shutil - dataset_path = os.environ['CM_DATASET_PATH'] preprocessed_dir = os.environ.get('CM_DATASET_PREPROCESSED_PATH', os.getcwd()) if os.environ.get('CM_DATASET_REFERENCE_PREPROCESSOR', '1') == "0": - #import generic_preprocess - #generic_preprocess.preprocess() + # import generic_preprocess + # generic_preprocess.preprocess() import preprocess_object_detection_dataset as pp pp.preprocess() else: @@ -26,22 +25,26 @@ image_width = int(os.environ.get('CM_DATASET_OPENIMAGES_RESIZE', 800)) threads = os.environ.get('CM_NUM_THREADS', os.cpu_count()) threads = os.environ.get('CM_NUM_PREPROCESS_THREADS', threads) - name="openimages-" + str(image_width) + "-retinanet" + name = "openimages-" + str(image_width) + "-retinanet" openimages.OpenImages(data_path=dataset_path, - image_list=dataset_list, - name=name, - image_format=img_format, - pre_process = dataset.pre_process_openimages_retinanet, - use_cache=True, - image_size=[image_width, image_width, 3], - count=count, - threads=threads, - preprocessed_dir=preprocessed_dir) + image_list=dataset_list, + name=name, + image_format=img_format, + pre_process=dataset.pre_process_openimages_retinanet, + use_cache=True, + image_size=[image_width, image_width, 3], + count=count, + threads=threads, + preprocessed_dir=preprocessed_dir) if os.environ["CM_DATASET_TYPE"] == "validation": - src_path=os.environ.get('CM_DATASET_ANNOTATIONS_DIR_PATH', os.path.join(dataset_path, "annotations")) - dest_path=os.path.join(preprocessed_dir, "annotations") + src_path = os.environ.get( + 'CM_DATASET_ANNOTATIONS_DIR_PATH', + os.path.join( + dataset_path, + "annotations")) + dest_path = os.path.join(preprocessed_dir, "annotations") if not os.path.exists(dest_path): shutil.copytree(src_path, dest_path) diff --git a/script/get-preprocessed-dataset-openorca/customize.py b/script/get-preprocessed-dataset-openorca/customize.py index 25b3acdc9a..7c042c839b 100644 --- a/script/get-preprocessed-dataset-openorca/customize.py +++ b/script/get-preprocessed-dataset-openorca/customize.py @@ -2,21 +2,27 @@ import os import shutil + def preprocess(i): env = i['env'] - if str(env.get('CM_DATASET_PREPROCESSED_BY_MLC','')).lower() in [ "yes", "1", "true" ]: + if str(env.get('CM_DATASET_PREPROCESSED_BY_MLC', '') + ).lower() in ["yes", "1", "true"]: run_dir = os.getcwd() if env.get('CM_DATASET_CALIBRATION', '') == "yes": - env['CM_DATASET_CALIBRATION_PATH'] = os.path.join(env['CM_OPENORCA_PREPROCESSED_ROOT'], "open_orca_gpt4_tokenized_llama.calibration_1000.pkl.gz") + env['CM_DATASET_CALIBRATION_PATH'] = os.path.join( + env['CM_OPENORCA_PREPROCESSED_ROOT'], + "open_orca_gpt4_tokenized_llama.calibration_1000.pkl.gz") env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_DATASET_CALIBRATION_PATH'] env['CM_DATASET_OPENORCA_CALIBRATION_PATH'] = env['CM_DATASET_CALIBRATION_PATH'] else: - env['CM_DATASET_PREPROCESSED_PATH'] = os.path.join(env['CM_OPENORCA_PREPROCESSED_ROOT'], "open_orca_gpt4_tokenized_llama.sampled_24576.pkl.gz") + env['CM_DATASET_PREPROCESSED_PATH'] = os.path.join( + env['CM_OPENORCA_PREPROCESSED_ROOT'], + "open_orca_gpt4_tokenized_llama.sampled_24576.pkl.gz") env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_DATASET_PREPROCESSED_PATH'] env['CM_DATASET_OPENORCA_PREPROCESSED_PATH'] = env['CM_DATASET_PREPROCESSED_PATH'] - #run_cmd = f"gunzip -k {env['CM_DATASET_PREPROCESSED_PATH']}" + # run_cmd = f"gunzip -k {env['CM_DATASET_PREPROCESSED_PATH']}" run_cmd = '' else: inference_src = env['CM_MLPERF_INFERENCE_SOURCE'] @@ -25,14 +31,24 @@ def preprocess(i): if env.get('CM_DATASET_CALIBRATION', '') == "yes": return {'return': 1, 'error': 'No raw preprocessing information is available for openorca calibration. Please use _mlcommons variation to use the MLCommons shared calibration dataset'} else: - env['CM_DATASET_PREPROCESSED_PATH'] = os.path.join(os.path.join(os.getcwd(), "processed-openorca", 'open_orca_gpt4_tokenized_llama.sampled_'+env['CM_DATASET_SIZE']+'.pkl')) - run_cmd = env['CM_PYTHON_BIN_WITH_PATH'] + ' processorca.py --dataset_pq_path=' + env['CM_DATASET_OPENORCA_PARQUET'] + ' --model_dir=' + model_dir +' --seqlen_limit=2048 --export_dir=' + os.path.join(os.getcwd(), "processed-openorca") + ' --num_total_samples=' + env['CM_DATASET_SIZE'] + env['CM_DATASET_PREPROCESSED_PATH'] = os.path.join( + os.path.join( + os.getcwd(), + "processed-openorca", + 'open_orca_gpt4_tokenized_llama.sampled_' + + env['CM_DATASET_SIZE'] + + '.pkl')) + run_cmd = env['CM_PYTHON_BIN_WITH_PATH'] + ' processorca.py --dataset_pq_path=' + env['CM_DATASET_OPENORCA_PARQUET'] + ' --model_dir=' + model_dir + \ + ' --seqlen_limit=2048 --export_dir=' + \ + os.path.join(os.getcwd(), "processed-openorca") + \ + ' --num_total_samples=' + env['CM_DATASET_SIZE'] env['CM_RUN_DIR'] = run_dir env['CM_RUN_CMD'] = run_cmd return {'return': 0} + def postprocess(i): env = i['env'] diff --git a/script/get-preprocessed-dataset-squad/customize.py b/script/get-preprocessed-dataset-squad/customize.py index 8e0ff47db5..1f72f139f6 100644 --- a/script/get-preprocessed-dataset-squad/customize.py +++ b/script/get-preprocessed-dataset-squad/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -14,10 +15,18 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') if env.get('CM_DATASET_SQUAD_CALIBRATION_SET', '') == "one": - env['DATASET_CALIBRATION_FILE'] = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], 'calibration', 'SQuAD-v1.1', 'bert_calibration_features.txt') + env['DATASET_CALIBRATION_FILE'] = os.path.join( + env['CM_MLPERF_INFERENCE_SOURCE'], + 'calibration', + 'SQuAD-v1.1', + 'bert_calibration_features.txt') env['DATASET_CALIBRATION_ID'] = 1 elif env.get('CM_DATASET_SQUAD_CALIBRATION_SET', '') == "two": - env['DATASET_CALIBRATION_FILE'] = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], 'calibration', 'SQuAD-v1.1', 'bert_calibration_qas_ids.txt') + env['DATASET_CALIBRATION_FILE'] = os.path.join( + env['CM_MLPERF_INFERENCE_SOURCE'], + 'calibration', + 'SQuAD-v1.1', + 'bert_calibration_qas_ids.txt') env['DATASET_CALIBRATION_ID'] = 2 else: env['DATASET_CALIBRATION_FILE'] = "''" @@ -32,7 +41,8 @@ def preprocess(i): env['+PYTHONPATH'].append(env['CM_MLPERF_INFERENCE_BERT_PATH']) - return {'return':0} + return {'return': 0} + def postprocess(i): @@ -42,11 +52,15 @@ def postprocess(i): if env.get('CM_DATASET_SQUAD_PACKED', '') != "yes": env['CM_DATASET_SQUAD_TOKENIZED_ROOT'] = cur if env.get('CM_DATASET_RAW', '') == "yes": - env['CM_DATASET_SQUAD_TOKENIZED_INPUT_IDS'] = os.path.join(cur, 'bert_tokenized_squad_v1_1_input_ids.raw') - env['CM_DATASET_SQUAD_TOKENIZED_SEGMENT_IDS'] = os.path.join(cur, 'bert_tokenized_squad_v1_1_segment_ids.raw') - env['CM_DATASET_SQUAD_TOKENIZED_INPUT_MASK'] = os.path.join(cur, 'bert_tokenized_squad_v1_1_input_mask.raw') + env['CM_DATASET_SQUAD_TOKENIZED_INPUT_IDS'] = os.path.join( + cur, 'bert_tokenized_squad_v1_1_input_ids.raw') + env['CM_DATASET_SQUAD_TOKENIZED_SEGMENT_IDS'] = os.path.join( + cur, 'bert_tokenized_squad_v1_1_segment_ids.raw') + env['CM_DATASET_SQUAD_TOKENIZED_INPUT_MASK'] = os.path.join( + cur, 'bert_tokenized_squad_v1_1_input_mask.raw') else: - env['CM_DATASET_SQUAD_TOKENIZED_PICKLE_FILE'] = os.path.join(cur, 'bert_tokenized_squad_v1_1.pickle') + env['CM_DATASET_SQUAD_TOKENIZED_PICKLE_FILE'] = os.path.join( + cur, 'bert_tokenized_squad_v1_1.pickle') env['CM_DATASET_SQUAD_TOKENIZED_MAX_SEQ_LENGTH'] = env['CM_DATASET_MAX_SEQ_LENGTH'] env['CM_DATASET_SQUAD_TOKENIZED_DOC_STRIDE'] = env['CM_DATASET_DOC_STRIDE'] @@ -56,7 +70,28 @@ def postprocess(i): with open("packed_filenames.txt", "w") as f: for dirname in os.listdir(cur): if os.path.isdir(dirname) and not dirname.startswith("_"): - f.write(os.path.join(cur, dirname, "input_ids.raw") + "," + os.path.join(cur, dirname, "input_mask.raw") + "," + os.path.join(cur, dirname, "segment_ids.raw") + "," + os.path.join(cur, dirname, "input_position_ids.raw")+ "\n") - env['CM_DATASET_SQUAD_TOKENIZED_PACKED_FILENAMES_FILE'] = os.path.join(cur, "packed_filenames.txt") - - return {'return':0} + f.write( + os.path.join( + cur, + dirname, + "input_ids.raw") + + "," + + os.path.join( + cur, + dirname, + "input_mask.raw") + + "," + + os.path.join( + cur, + dirname, + "segment_ids.raw") + + "," + + os.path.join( + cur, + dirname, + "input_position_ids.raw") + + "\n") + env['CM_DATASET_SQUAD_TOKENIZED_PACKED_FILENAMES_FILE'] = os.path.join( + cur, "packed_filenames.txt") + + return {'return': 0} diff --git a/script/get-qaic-apps-sdk/customize.py b/script/get-qaic-apps-sdk/customize.py index b84d58b178..5fd343f71c 100644 --- a/script/get-qaic-apps-sdk/customize.py +++ b/script/get-qaic-apps-sdk/customize.py @@ -2,6 +2,7 @@ import os import xml.etree.ElementTree as et + def preprocess(i): os_info = i['os_info'] @@ -14,26 +15,29 @@ def preprocess(i): apps_sdk_path = None - if env.get('CM_INPUT','').strip() != '': + if env.get('CM_INPUT', '').strip() != '': path = env['CM_INPUT'] if os.path.exists(os.path.join(path, "exec", "qaic-exec")): apps_sdk_path = path else: - return {'return':1, 'error': 'exec/qaic-exec not found in the input path (--input)'} + return { + 'return': 1, 'error': 'exec/qaic-exec not found in the input path (--input)'} else: path = "/opt/qti-aic/" if os.path.exists(os.path.join(path, "exec", "qaic-exec")): apps_sdk_path = path if not apps_sdk_path: - return {'return':1, 'error': f'qaic-exec not found in the default path: {path}'} + return {'return': 1, + 'error': f'qaic-exec not found in the default path: {path}'} env['CM_QAIC_APPS_SDK_PATH'] = path env['CM_QAIC_EXEC_PATH'] = os.path.join(path, "exec", "qaic-exec") quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def detect_version(i): @@ -54,13 +58,14 @@ def detect_version(i): if child2.tag == "build_id": build_id = child2.text if build_id: - version=version+"."+build_id + version = version + "." + build_id if not version: - return {'return':1, 'error': f'qaic apps sdk version info not found'} + return {'return': 1, 'error': f'qaic apps sdk version info not found'} + + print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + return {'return': 0, 'version': version} - print (i['recursion_spaces'] + ' Detected version: {}'.format(version)) - return {'return':0, 'version':version} def postprocess(i): @@ -77,11 +82,11 @@ def postprocess(i): env['+PATH'].append(os.path.dirname(env['CM_QAIC_EXEC_PATH'])) paths = [ - "+C_INCLUDE_PATH", - "+CPLUS_INCLUDE_PATH", - "+LD_LIBRARY_PATH", - "+DYLD_FALLBACK_LIBRARY_PATH" - ] + "+C_INCLUDE_PATH", + "+CPLUS_INCLUDE_PATH", + "+LD_LIBRARY_PATH", + "+DYLD_FALLBACK_LIBRARY_PATH" + ] for key in paths: env[key] = [] @@ -97,8 +102,11 @@ def postprocess(i): env['+C_INCLUDE_PATH'].append(inc_path) env['+CPLUS_INCLUDE_PATH'].append(inc_path) - - lib_path = os.path.join(env['CM_QAIC_APPS_SDK_PATH'], "dev", "lib", "x86_64") + lib_path = os.path.join( + env['CM_QAIC_APPS_SDK_PATH'], + "dev", + "lib", + "x86_64") if os.path.exists(lib_path): lib_paths.append(lib_path) @@ -106,4 +114,4 @@ def postprocess(i): env['+LD_LIBRARY_PATH'].append(lib_path) env['+DYLD_FALLBACK_LIBRARY_PATH'].append(lib_path) - return {'return':0, 'version': version} + return {'return': 0, 'version': version} diff --git a/script/get-qaic-platform-sdk/customize.py b/script/get-qaic-platform-sdk/customize.py index 5a68188bd5..33e9548ba7 100644 --- a/script/get-qaic-platform-sdk/customize.py +++ b/script/get-qaic-platform-sdk/customize.py @@ -2,6 +2,7 @@ import os import xml.etree.ElementTree as et + def preprocess(i): os_info = i['os_info'] @@ -14,19 +15,21 @@ def preprocess(i): platform_sdk_path = None - if env.get('CM_INPUT','').strip() != '': + if env.get('CM_INPUT', '').strip() != '': path = env['CM_INPUT'] if os.path.exists(os.path.join(path, "exec", "qaic-runner")): platform_sdk_path = path else: - return {'return':1, 'error': 'exec/qaic-runner not found in the input path (--input)'} + return { + 'return': 1, 'error': 'exec/qaic-runner not found in the input path (--input)'} else: path = "/opt/qti-aic/" if os.path.exists(os.path.join(path, "exec", "qaic-runner")): platform_sdk_path = path if not platform_sdk_path: - return {'return':1, 'error': f'qaic-runner not found in the default path: {path}'} + return {'return': 1, + 'error': f'qaic-runner not found in the default path: {path}'} env['CM_QAIC_PLATFORM_SDK_PATH'] = path env['CM_QAIC_RUNNER_PATH'] = os.path.join(path, "exec", "qaic-runner") @@ -34,7 +37,8 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def detect_version(i): @@ -55,13 +59,14 @@ def detect_version(i): if child2.tag == "build_id": build_id = child2.text if build_id: - version=version+"."+build_id + version = version + "." + build_id if not version: - return {'return':1, 'error': f'qaic platform sdk version info not found'} + return {'return': 1, 'error': f'qaic platform sdk version info not found'} + + print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + return {'return': 0, 'version': version} - print (i['recursion_spaces'] + ' Detected version: {}'.format(version)) - return {'return':0, 'version':version} def postprocess(i): @@ -78,11 +83,11 @@ def postprocess(i): env['+PATH'].append(os.path.dirname(env['CM_QAIC_RUNNER_PATH'])) paths = [ - "+C_INCLUDE_PATH", - "+CPLUS_INCLUDE_PATH", - "+LD_LIBRARY_PATH", - "+DYLD_FALLBACK_LIBRARY_PATH" - ] + "+C_INCLUDE_PATH", + "+CPLUS_INCLUDE_PATH", + "+LD_LIBRARY_PATH", + "+DYLD_FALLBACK_LIBRARY_PATH" + ] for key in paths: env[key] = [] @@ -98,8 +103,11 @@ def postprocess(i): env['+C_INCLUDE_PATH'].append(inc_path) env['+CPLUS_INCLUDE_PATH'].append(inc_path) - - lib_path = os.path.join(env['CM_QAIC_PLATFORM_SDK_PATH'], "dev", "lib", env['CM_HOST_PLATFORM_FLAVOR']) + lib_path = os.path.join( + env['CM_QAIC_PLATFORM_SDK_PATH'], + "dev", + "lib", + env['CM_HOST_PLATFORM_FLAVOR']) if os.path.exists(lib_path): lib_paths.append(lib_path) @@ -107,4 +115,4 @@ def postprocess(i): env['+LD_LIBRARY_PATH'].append(lib_path) env['+DYLD_FALLBACK_LIBRARY_PATH'].append(lib_path) - return {'return':0, 'version': version} + return {'return': 0, 'version': version} diff --git a/script/get-spec-ptd/customize.py b/script/get-spec-ptd/customize.py index 250ddd887b..b4c949179d 100644 --- a/script/get-spec-ptd/customize.py +++ b/script/get-spec-ptd/customize.py @@ -2,11 +2,12 @@ import os import shutil + def preprocess(i): os_info = i['os_info'] - return {'return':0} + return {'return': 0} def postprocess(i): @@ -19,7 +20,8 @@ def postprocess(i): else: binary_name = "ptd-linux-x86" if 'CM_MLPERF_PTD_PATH' not in env: - env['CM_MLPERF_PTD_PATH'] = os.path.join(env['CM_MLPERF_POWER_SOURCE'], 'inference_v1.0', binary_name) + env['CM_MLPERF_PTD_PATH'] = os.path.join( + env['CM_MLPERF_POWER_SOURCE'], 'inference_v1.0', binary_name) env['CM_SPEC_PTD_PATH'] = env['CM_MLPERF_PTD_PATH'] - return {'return':0} + return {'return': 0} diff --git a/script/get-tvm/customize.py b/script/get-tvm/customize.py index fba65ec4b7..b7995264ba 100644 --- a/script/get-tvm/customize.py +++ b/script/get-tvm/customize.py @@ -1,13 +1,15 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} + + return {'return': 0} - return {'return':0} def postprocess(i): @@ -15,23 +17,22 @@ def postprocess(i): env = i['env'] if env.get('CM_TVM_PIP_INSTALL', '') == "yes": - return {'return':0} - + return {'return': 0} tvm_home = env['TVM_HOME'] # 20221024: we save and restore env in the main script and can clean env here for determinism # if '+PYTHONPATH' not in env: env['+PYTHONPATH']=[] - env['+PYTHONPATH']=[] - - env['+PYTHONPATH'].append(os.path.join(tvm_home,'python')) + env['+PYTHONPATH'] = [] + env['+PYTHONPATH'].append(os.path.join(tvm_home, 'python')) # Prepare paths - for key in ['+C_INCLUDE_PATH', '+CPLUS_INCLUDE_PATH', '+LD_LIBRARY_PATH', '+DYLD_FALLBACK_LIBRARY_PATH']: + for key in ['+C_INCLUDE_PATH', '+CPLUS_INCLUDE_PATH', + '+LD_LIBRARY_PATH', '+DYLD_FALLBACK_LIBRARY_PATH']: env[key] = [] - ## Include + # Include include_path = os.path.join(tvm_home, 'include') if os.path.isdir(include_path): if os_info['platform'] != 'windows': @@ -40,11 +41,10 @@ def postprocess(i): env['CM_TVM_PATH_INCLUDE'] = include_path - ## Lib + # Lib lib_path = os.path.join(tvm_home, 'build') env['+LD_LIBRARY_PATH'].append(lib_path) env['+DYLD_FALLBACK_LIBRARY_PATH'].append(lib_path) env['CM_TVM_PATH_LIB'] = lib_path - - return {'return':0} + return {'return': 0} diff --git a/script/get-xilinx-sdk/customize.py b/script/get-xilinx-sdk/customize.py index 02e31c620f..52922d29c0 100644 --- a/script/get-xilinx-sdk/customize.py +++ b/script/get-xilinx-sdk/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -15,18 +16,19 @@ def preprocess(i): file_path = env.get("CM_XILINX_SDK_BIN_PATH") if not file_path or not os.path.exists(file_path): - return {'return':1, 'error': 'FILE_PATH does not exist'} + return {'return': 1, 'error': 'FILE_PATH does not exist'} bin_folder_path = os.path.dirname(file_path) if '+PATH' in env: env['+PATH'].append(bin_foler_path) else: - env['+PATH'] = [ bin_folder_path ] + env['+PATH'] = [bin_folder_path] + + return {'return': 0} - return {'return':0} def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/get-zendnn/customize.py b/script/get-zendnn/customize.py index d9918a266d..5310eea521 100644 --- a/script/get-zendnn/customize.py +++ b/script/get-zendnn/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -18,10 +19,11 @@ def preprocess(i): env['ZENDNN_SRC_PATH'] = env['CM_GIT_REPO_CHECKOUT_PATH'] - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/get-zephyr-sdk/customize.py b/script/get-zephyr-sdk/customize.py index 87619e7a01..d0b05d6757 100644 --- a/script/get-zephyr-sdk/customize.py +++ b/script/get-zephyr-sdk/customize.py @@ -1,15 +1,17 @@ from cmind import utils import os + def preprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} def postprocess(i): env = i['env'] env['ZEPHYR_TOOLCHAIN_VARIANT'] = "zephyr" - env['ZEPHYR_SDK_INSTALL_DIR'] = os.path.join(os.getcwd(), "zephyr-sdk-" + env['CM_ZEPHYR_SDK_VERSION']) + env['ZEPHYR_SDK_INSTALL_DIR'] = os.path.join( + os.getcwd(), "zephyr-sdk-" + env['CM_ZEPHYR_SDK_VERSION']) - return {'return':0} + return {'return': 0} diff --git a/script/get-zephyr/customize.py b/script/get-zephyr/customize.py index c157b165ce..74b8a9fec4 100644 --- a/script/get-zephyr/customize.py +++ b/script/get-zephyr/customize.py @@ -1,12 +1,13 @@ from cmind import utils import os + def preprocess(i): env = i['env'] if '+PATH' not in env: env['+PATH'] = [] env['+PATH'].append("$HOME/.local/bin") - return {'return':0} + return {'return': 0} def postprocess(i): @@ -14,4 +15,4 @@ def postprocess(i): env = i['env'] env['CM_ZEPHYR_DIR'] = os.path.join(os.getcwd(), "zephyr") - return {'return':0} + return {'return': 0} diff --git a/script/gui/tests/generate_password.py b/script/gui/tests/generate_password.py index 145a46dbd3..a29f7cd637 100644 --- a/script/gui/tests/generate_password.py +++ b/script/gui/tests/generate_password.py @@ -1,8 +1,8 @@ import bcrypt -#salt = bcrypt.gensalt() +# salt = bcrypt.gensalt() # TBD: temporal hack to demo password protection for experiments -#salt = bcrypt.gensalt() +# salt = bcrypt.gensalt() pwd = input('Password: ') pwd = pwd.strip() @@ -10,4 +10,4 @@ password_salt = b'$2b$12$ionIRWe5Ft7jkn4y/7C6/e' password_hash2 = bcrypt.hashpw(pwd.encode('utf-8'), password_salt) -print ('"password_hash":"{}"'.format(password_hash2.decode('utf-8'))) +print('"password_hash":"{}"'.format(password_hash2.decode('utf-8'))) diff --git a/script/install-apt-package/customize.py b/script/install-apt-package/customize.py index b8e750ecb9..21e27eedaa 100644 --- a/script/install-apt-package/customize.py +++ b/script/install-apt-package/customize.py @@ -2,6 +2,7 @@ import os import re + def preprocess(i): os_info = i['os_info'] @@ -12,13 +13,15 @@ def preprocess(i): install_cmd = env.get('CM_HOST_OS_PACKAGE_MANAGER_INSTALL_CMD') if not install_cmd: - return {'return': 1, 'error': 'Package manager installation command not detected for the given OS'} + return { + 'return': 1, 'error': 'Package manager installation command not detected for the given OS'} sudo = env.get('CM_SUDO', '') - env['CM_APT_INSTALL_CMD'] = sudo + ' ' +install_cmd + ' ' + package_name + env['CM_APT_INSTALL_CMD'] = sudo + ' ' + install_cmd + ' ' + package_name - if env.get('CM_APT_CHECK_CMD', '') != '' and env['CM_APT_INSTALL_CMD'] != '': + if env.get('CM_APT_CHECK_CMD', + '') != '' and env['CM_APT_INSTALL_CMD'] != '': env['CM_APT_INSTALL_CMD'] = f"""{env['CM_APT_CHECK_CMD']} || {env['CM_APT_INSTALL_CMD']}""" - return {'return':0} + return {'return': 0} diff --git a/script/install-aws-cli/customize.py b/script/install-aws-cli/customize.py index df2744ac4d..032a4f3bf2 100644 --- a/script/install-aws-cli/customize.py +++ b/script/install-aws-cli/customize.py @@ -1,12 +1,13 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] @@ -14,4 +15,4 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] - return {'return':0} + return {'return': 0} diff --git a/script/install-cuda-package-manager/customize.py b/script/install-cuda-package-manager/customize.py index 002e85e2ef..fb84652e3d 100644 --- a/script/install-cuda-package-manager/customize.py +++ b/script/install-cuda-package-manager/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -10,7 +11,6 @@ def preprocess(i): automation = i['automation'] recursion_spaces = i['recursion_spaces'] - env['CM_GET_DEPENDENT_CACHED_PATH'] = os.getcwd() - + env['CM_GET_DEPENDENT_CACHED_PATH'] = os.getcwd() - return {'return':0} + return {'return': 0} diff --git a/script/install-cuda-prebuilt/customize.py b/script/install-cuda-prebuilt/customize.py index ac20aca719..db49f81b32 100644 --- a/script/install-cuda-prebuilt/customize.py +++ b/script/install-cuda-prebuilt/customize.py @@ -1,13 +1,14 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] env = i['env'] - if str(env.get('CUDA_SKIP_SUDO','')).lower() == 'true': + if str(env.get('CUDA_SKIP_SUDO', '')).lower() == 'true': env['CM_SUDO'] = '' meta = i['meta'] @@ -16,7 +17,8 @@ def preprocess(i): if version not in env.get('CM_CUDA_LINUX_FILENAME', ''): supported_versions = list(meta['versions'].keys()) - return {'return': 1, 'error': "Only CUDA versions {} are supported now".format(', '.join(supported_versions))} + return {'return': 1, 'error': "Only CUDA versions {} are supported now".format( + ', '.join(supported_versions))} install_prefix = env.get('CM_CUDA_INSTALL_PREFIX', os.getcwd()) @@ -30,20 +32,24 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] nvcc_bin = "nvcc" - env['WGET_URL']="https://developer.download.nvidia.com/compute/cuda/"+env['CM_VERSION']+"/local_installers/"+env['CM_CUDA_LINUX_FILENAME'] + env['WGET_URL'] = "https://developer.download.nvidia.com/compute/cuda/" + \ + env['CM_VERSION'] + "/local_installers/" + \ + env['CM_CUDA_LINUX_FILENAME'] extra_options = env.get('CUDA_ADDITIONAL_INSTALL_OPTIONS', '') - if env.get('CM_CUDA_INSTALL_DRIVER','') == "yes": + if env.get('CM_CUDA_INSTALL_DRIVER', '') == "yes": extra_options += " --driver" env['CUDA_ADDITIONAL_INSTALL_OPTIONS'] = extra_options env['CM_CUDA_INSTALLED_PATH'] = os.path.join(install_prefix, 'install') - env['CM_NVCC_BIN_WITH_PATH'] = os.path.join(install_prefix, 'install', 'bin', nvcc_bin) - env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_NVCC_BIN_WITH_PATH'] + env['CM_NVCC_BIN_WITH_PATH'] = os.path.join( + install_prefix, 'install', 'bin', nvcc_bin) + env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_NVCC_BIN_WITH_PATH'] env['CM_CUDA_EXTRA_INSTALL_ARGS'] = extra_install_args - # Set CUDA_RUN_FILE_LOCAL_PATH to empty if not set for backwards compatibility in download file - env['CUDA_RUN_FILE_LOCAL_PATH'] = env.get('CUDA_RUN_FILE_LOCAL_PATH','') + # Set CUDA_RUN_FILE_LOCAL_PATH to empty if not set for backwards + # compatibility in download file + env['CUDA_RUN_FILE_LOCAL_PATH'] = env.get('CUDA_RUN_FILE_LOCAL_PATH', '') - return {'return':0} + return {'return': 0} diff --git a/script/install-diffusers-from-src/customize.py b/script/install-diffusers-from-src/customize.py index 0e1ca24f5c..6d5e7f9db3 100644 --- a/script/install-diffusers-from-src/customize.py +++ b/script/install-diffusers-from-src/customize.py @@ -1,21 +1,22 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] - automation = i['automation'] recursion_spaces = i['recursion_spaces'] - return {'return':0} + return {'return': 0} + def postprocess(i): - return {'return':0} + return {'return': 0} diff --git a/script/install-gcc-src/customize.py b/script/install-gcc-src/customize.py index caff463edc..a368cdfb8d 100644 --- a/script/install-gcc-src/customize.py +++ b/script/install-gcc-src/customize.py @@ -1,12 +1,13 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] @@ -14,15 +15,16 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] - need_version = env.get('CM_VERSION','') + need_version = env.get('CM_VERSION', '') if need_version == '': - return {'return':1, 'error':'internal problem - CM_VERSION is not defined in env'} + return {'return': 1, + 'error': 'internal problem - CM_VERSION is not defined in env'} - print (recursion_spaces + ' # Requested version: {}'.format(need_version)) + print(recursion_spaces + ' # Requested version: {}'.format(need_version)) if 'CM_GIT_CHECKOUT' not in env: env['CM_GIT_CHECKOUT'] = 'releases/gcc-' + need_version env['CM_GCC_INSTALLED_PATH'] = os.path.join(os.getcwd(), 'install', 'bin') - return {'return':0} + return {'return': 0} diff --git a/script/install-generic-conda-package/customize.py b/script/install-generic-conda-package/customize.py index 2e6486a54d..5310923d66 100644 --- a/script/install-generic-conda-package/customize.py +++ b/script/install-generic-conda-package/customize.py @@ -2,6 +2,7 @@ import os import cmind as cm + def preprocess(i): os_info = i['os_info'] @@ -15,22 +16,22 @@ def preprocess(i): install_cmd = env['CM_CONDA_BIN_WITH_PATH'] + " install -y " if env.get('CM_CONDA_PKG_SRC', '') != '': - install_cmd += " -c "+env['CM_CONDA_PKG_SRC'] + " " + install_cmd += " -c " + env['CM_CONDA_PKG_SRC'] + " " install_cmd += package_name install_cmd += version_string env['CM_CONDA_PKG_INSTALL_CMD'] = install_cmd + return {'return': 0} - return {'return':0} def detect_version(i): # TBD - print (i['recursion_spaces'] + ' Detected version: {}'.format(version)) + print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) - return {'return':0, 'version':version} + return {'return': 0, 'version': version} def postprocess(i): @@ -39,6 +40,7 @@ def postprocess(i): version = env.get('CM_VERSION', '') if env['CM_CONDA_PKG_NAME'] == "python": - env['CM_PYTHON_BIN_WITH_PATH'] = os.path.join(os.path.dirname(env['CM_CONDA_BIN_WITH_PATH']), "python") + env['CM_PYTHON_BIN_WITH_PATH'] = os.path.join( + os.path.dirname(env['CM_CONDA_BIN_WITH_PATH']), "python") - return {'return':0, 'version': version} + return {'return': 0, 'version': version} diff --git a/script/install-gflags-from-src/customize.py b/script/install-gflags-from-src/customize.py index 216af841e8..030b0b8b45 100644 --- a/script/install-gflags-from-src/customize.py +++ b/script/install-gflags-from-src/customize.py @@ -1,12 +1,13 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] @@ -14,13 +15,14 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] env['CM_GFLAGS_BUILD_PATH'] = os.path.join(os.getcwd(), "gflags", "build") - env['CM_DEPENDENT_CACHED_PATH' ] = env['CM_GFLAGS_BUILD_PATH'] + env['CM_DEPENDENT_CACHED_PATH'] = env['CM_GFLAGS_BUILD_PATH'] - return {'return':0} + return {'return': 0} diff --git a/script/install-gflags/customize.py b/script/install-gflags/customize.py index 65872c79a0..d9caffb60c 100644 --- a/script/install-gflags/customize.py +++ b/script/install-gflags/customize.py @@ -1,12 +1,13 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] @@ -14,15 +15,17 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] - need_version = env.get('CM_VERSION','') + need_version = env.get('CM_VERSION', '') if need_version == '': - return {'return':1, 'error':'internal problem - CM_VERSION is not defined in env'} + return {'return': 1, + 'error': 'internal problem - CM_VERSION is not defined in env'} + + print(recursion_spaces + ' # Requested version: {}'.format(need_version)) - print (recursion_spaces + ' # Requested version: {}'.format(need_version)) + return {'return': 0} - return {'return':0} def postprocess(i): inp = i['input'] env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/install-github-cli/customize.py b/script/install-github-cli/customize.py index cd7d65a35b..5079c4d741 100644 --- a/script/install-github-cli/customize.py +++ b/script/install-github-cli/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -10,4 +11,4 @@ def preprocess(i): env['CM_TMP_PATH'] = os.path.join(os.getcwd(), 'install', 'bin') env['CM_TMP_FAIL_IF_NOT_FOUND'] = 'yes' - return {'return':0} + return {'return': 0} diff --git a/script/install-ipex-from-src/customize.py b/script/install-ipex-from-src/customize.py index d146b70552..d257493046 100644 --- a/script/install-ipex-from-src/customize.py +++ b/script/install-ipex-from-src/customize.py @@ -1,12 +1,13 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] @@ -15,19 +16,25 @@ def preprocess(i): if env.get('CM_USE_LLVM_FOR_IPEX', '') == 'yes': env['DNNL_GRAPH_BUILD_COMPILER_BACKEND'] = 1 env['USE_LLVM'] = env['CM_LLVM_INSTALLED_PATH'] - env['LLVM_DIR'] = os.path.join(env['CM_LLVM_INSTALLED_PATH'], "lib", "cmake", "llvm") + env['LLVM_DIR'] = os.path.join( + env['CM_LLVM_INSTALLED_PATH'], "lib", "cmake", "llvm") - run_cmd="python setup.py clean && python setup.py install" + run_cmd = "python setup.py clean && python setup.py install" env['CM_RUN_CMD'] = run_cmd - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] env['CM_IPEX_BUILD_PATH'] = os.path.join(os.getcwd(), "ipex_src", "build") - env['CM_IPEX_INSTALLED_PATH'] = os.path.join(env['CM_IPEX_BUILD_PATH'], "Release", "packages", "intel_extension_for_pytorch") + env['CM_IPEX_INSTALLED_PATH'] = os.path.join( + env['CM_IPEX_BUILD_PATH'], + "Release", + "packages", + "intel_extension_for_pytorch") env['CM_DEPENDENT_CACHED_PATH'] = env['CM_IPEX_INSTALLED_PATH'] - return {'return':0} + return {'return': 0} diff --git a/script/install-llvm-src/customize.py b/script/install-llvm-src/customize.py index b1bb23250f..5a359ac7e0 100644 --- a/script/install-llvm-src/customize.py +++ b/script/install-llvm-src/customize.py @@ -1,12 +1,13 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] @@ -23,8 +24,8 @@ def preprocess(i): env['CM_REQUIRE_INSTALL'] = 'yes' i['run_script_input']['script_name'] = "install-llvm-16-intel-mlperf-inference" clang_file_name = "llvm-link" - #env['USE_LLVM'] = install_prefix - #env['LLVM_DIR'] = os.path.join(env['USE_LLVM'], "lib", "cmake", "llvm") + # env['USE_LLVM'] = install_prefix + # env['LLVM_DIR'] = os.path.join(env['USE_LLVM'], "lib", "cmake", "llvm") else: if env.get('CM_LLVM_ENABLE_RUNTIMES', '') != '': enable_runtimes = env['CM_LLVM_ENABLE_RUNTIMES'].replace(":", ";") @@ -38,19 +39,24 @@ def preprocess(i): llvm_build_type = env['CM_LLVM_BUILD_TYPE'] - cmake_cmd = "cmake " + os.path.join(env["CM_LLVM_SRC_REPO_PATH"], "llvm") + " -GNinja -DCMAKE_BUILD_TYPE="+llvm_build_type + " -DLLVM_ENABLE_PROJECTS="+ enable_projects+ " -DLLVM_ENABLE_RUNTIMES='"+enable_runtimes + "' -DCMAKE_INSTALL_PREFIX=" + install_prefix + " -DLLVM_ENABLE_RTTI=ON -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD=X86 " + extra_cmake_options + cmake_cmd = "cmake " + os.path.join(env["CM_LLVM_SRC_REPO_PATH"], "llvm") + " -GNinja -DCMAKE_BUILD_TYPE=" + llvm_build_type + " -DLLVM_ENABLE_PROJECTS=" + enable_projects + " -DLLVM_ENABLE_RUNTIMES='" + \ + enable_runtimes + "' -DCMAKE_INSTALL_PREFIX=" + install_prefix + \ + " -DLLVM_ENABLE_RTTI=ON -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD=X86 " + \ + extra_cmake_options env['CM_LLVM_CMAKE_CMD'] = cmake_cmd - need_version = env.get('CM_VERSION','') + need_version = env.get('CM_VERSION', '') - #print(cmake_cmd) + # print(cmake_cmd) env['CM_LLVM_INSTALLED_PATH'] = install_prefix - env['CM_LLVM_CLANG_BIN_WITH_PATH'] = os.path.join(env['CM_LLVM_INSTALLED_PATH'], "bin", clang_file_name) + env['CM_LLVM_CLANG_BIN_WITH_PATH'] = os.path.join( + env['CM_LLVM_INSTALLED_PATH'], "bin", clang_file_name) + + # env['+PATH'] = [] + return {'return': 0} - #env['+PATH'] = [] - return {'return':0} def postprocess(i): @@ -59,11 +65,12 @@ def postprocess(i): env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_LLVM_CLANG_BIN_WITH_PATH'] if env.get('CM_LLVM_CONDA_ENV', '') != "yes": - # We don't need to check default paths here because we force install to cache - env['+PATH'] = [ os.path.join(env['CM_LLVM_INSTALLED_PATH'], "bin") ] + # We don't need to check default paths here because we force install to + # cache + env['+PATH'] = [os.path.join(env['CM_LLVM_INSTALLED_PATH'], "bin")] path_include = os.path.join(env['CM_LLVM_INSTALLED_PATH'], 'include') if os.path.isdir(path_include): - env['+C_INCLUDE_PATH'] = [ path_include ] + env['+C_INCLUDE_PATH'] = [path_include] - return {'return':0} + return {'return': 0} diff --git a/script/install-mlperf-logging-from-src/customize.py b/script/install-mlperf-logging-from-src/customize.py index d12f9b3e1d..273999d460 100644 --- a/script/install-mlperf-logging-from-src/customize.py +++ b/script/install-mlperf-logging-from-src/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,10 +14,11 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/install-nccl-libs/customize.py b/script/install-nccl-libs/customize.py index d12f9b3e1d..273999d460 100644 --- a/script/install-nccl-libs/customize.py +++ b/script/install-nccl-libs/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,10 +14,11 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/install-numactl-from-src/customize.py b/script/install-numactl-from-src/customize.py index 5c1ee2674f..5605e99b00 100644 --- a/script/install-numactl-from-src/customize.py +++ b/script/install-numactl-from-src/customize.py @@ -1,16 +1,17 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] - run_cmd="python setup.py install" + run_cmd = "python setup.py install" env['CM_RUN_CMD'] = run_cmd @@ -20,4 +21,4 @@ def preprocess(i): env['+PATH'] = [] - return {'return':0} + return {'return': 0} diff --git a/script/install-onednn-from-src/customize.py b/script/install-onednn-from-src/customize.py index 95d18ff02e..facd107df1 100644 --- a/script/install-onednn-from-src/customize.py +++ b/script/install-onednn-from-src/customize.py @@ -1,16 +1,17 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] - run_cmd="" + run_cmd = "" env['CM_RUN_CMD'] = run_cmd env['CM_ONEDNN_INSTALLED_PATH'] = os.path.join(os.getcwd(), "onednn") @@ -22,10 +23,11 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/install-onnxruntime-from-src/customize.py b/script/install-onnxruntime-from-src/customize.py index be854b226b..0f11967a0e 100644 --- a/script/install-onnxruntime-from-src/customize.py +++ b/script/install-onnxruntime-from-src/customize.py @@ -1,16 +1,17 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] - run_cmd="./build.sh --config RelWithDebInfo --build_wheel --parallel --allow_running_as_root --skip_tests " + run_cmd = "./build.sh --config RelWithDebInfo --build_wheel --parallel --allow_running_as_root --skip_tests " if env.get('CM_ONNXRUNTIME_GPU', '') == "yes": cuda_home = env['CUDA_HOME'] @@ -19,4 +20,4 @@ def preprocess(i): env['CM_RUN_DIR'] = env['CM_ONNXRUNTIME_SRC_REPO_PATH'] env['CM_RUN_CMD'] = run_cmd - return {'return':0} + return {'return': 0} diff --git a/script/install-opencv-from-src/customize.py b/script/install-opencv-from-src/customize.py index 0e5521f943..bc4ebf61fc 100644 --- a/script/install-opencv-from-src/customize.py +++ b/script/install-opencv-from-src/customize.py @@ -1,12 +1,13 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] @@ -14,13 +15,14 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] env['CM_OPENCV_BUILD_PATH'] = os.path.join(os.getcwd(), "opencv", "build") - env['CM_DEPENDENT_CACHED_PATH' ] = env['CM_OPENCV_BUILD_PATH'] + env['CM_DEPENDENT_CACHED_PATH'] = env['CM_OPENCV_BUILD_PATH'] - return {'return':0} + return {'return': 0} diff --git a/script/install-openssl/customize.py b/script/install-openssl/customize.py index e6163a0f5e..23ece8afb5 100644 --- a/script/install-openssl/customize.py +++ b/script/install-openssl/customize.py @@ -1,12 +1,13 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] @@ -14,20 +15,27 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] - need_version = env.get('CM_VERSION','') + need_version = env.get('CM_VERSION', '') if need_version == '': - return {'return':1, 'error':'internal problem - CM_VERSION is not defined in env'} + return {'return': 1, + 'error': 'internal problem - CM_VERSION is not defined in env'} + + print(recursion_spaces + ' # Requested version: {}'.format(need_version)) - print (recursion_spaces + ' # Requested version: {}'.format(need_version)) + return {'return': 0} - return {'return':0} def postprocess(i): inp = i['input'] env = i['env'] tags = inp['tags'] tag_list = tags.split(",") - install_path = os.path.join(os.getcwd(), 'openssl-'+env['CM_VERSION']+'g', 'install') + install_path = os.path.join( + os.getcwd(), + 'openssl-' + + env['CM_VERSION'] + + 'g', + 'install') path_lib = os.path.join(install_path, 'lib') if '+LD_LIBRARY_PATH' not in env: env['+LD_LIBRARY_PATH'] = [] @@ -36,4 +44,4 @@ def postprocess(i): path_bin = os.path.join(install_path, 'bin') env['CM_OPENSSL_INSTALLED_PATH'] = path_bin env['CM_OPENSSL_BIN_WITH_PATH'] = os.path.join(path_bin, bin_name) - return {'return':0} + return {'return': 0} diff --git a/script/install-python-src/customize.py b/script/install-python-src/customize.py index a7025a6cf2..25edf22f49 100644 --- a/script/install-python-src/customize.py +++ b/script/install-python-src/customize.py @@ -1,12 +1,13 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] @@ -14,17 +15,19 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] - need_version = env.get('CM_VERSION','') + need_version = env.get('CM_VERSION', '') if need_version == '': - return {'return':1, 'error':'internal problem - CM_VERSION is not defined in env'} + return {'return': 1, + 'error': 'internal problem - CM_VERSION is not defined in env'} - print (recursion_spaces + ' # Requested version: {}'.format(need_version)) + print(recursion_spaces + ' # Requested version: {}'.format(need_version)) path_bin = os.path.join(os.getcwd(), 'install', 'bin') env['CM_PYTHON_INSTALLED_PATH'] = path_bin - return {'return':0} + return {'return': 0} + def postprocess(i): @@ -32,15 +35,17 @@ def postprocess(i): variation_tags = i['variation_tags'] path_lib = os.path.join(os.getcwd(), 'install', 'lib') - env['+LD_LIBRARY_PATH'] = [ path_lib ] + env['+LD_LIBRARY_PATH'] = [path_lib] - env['CM_GET_DEPENDENT_CACHED_PATH'] = os.getcwd() + env['CM_GET_DEPENDENT_CACHED_PATH'] = os.getcwd() - env['CM_PYTHON_BIN_WITH_PATH'] = os.path.join(env['CM_PYTHON_INSTALLED_PATH'], 'python3') + env['CM_PYTHON_BIN_WITH_PATH'] = os.path.join( + env['CM_PYTHON_INSTALLED_PATH'], 'python3') - # We don't need to check default paths here because we force install to cache + # We don't need to check default paths here because we force install to + # cache env['+PATH'] = [env['CM_PYTHON_INSTALLED_PATH']] path_include = os.path.join(os.getcwd(), 'install', 'include') - env['+C_INCLUDE_PATH'] = [ path_include ] + env['+C_INCLUDE_PATH'] = [path_include] - return {'return':0} + return {'return': 0} diff --git a/script/install-pytorch-from-src/customize.py b/script/install-pytorch-from-src/customize.py index 67f2673d33..d44872d7a8 100644 --- a/script/install-pytorch-from-src/customize.py +++ b/script/install-pytorch-from-src/customize.py @@ -1,23 +1,24 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] if env.get('CM_MLPERF_INFERENCE_INTEL_LANGUAGE_MODEL', '') == "yes": i['run_script_input']['script_name'] = "run-intel-mlperf-inference-v3_1" - run_cmd="CC=clang CXX=clang++ USE_CUDA=OFF python -m pip install -e . " + run_cmd = "CC=clang CXX=clang++ USE_CUDA=OFF python -m pip install -e . " env['CM_RUN_CMD'] = run_cmd - elif env.get('CM_MLPERF_INFERENCE_INTEL_MODEL', '') in [ "resnet50", "retinanet" ]: + elif env.get('CM_MLPERF_INFERENCE_INTEL_MODEL', '') in ["resnet50", "retinanet"]: i['run_script_input']['script_name'] = "run-intel-mlperf-inference-vision" - run_cmd=f"CC={env['CM_C_COMPILER_WITH_PATH']} CXX={env['CM_CXX_COMPILER_WITH_PATH']} USE_CUDA=OFF python -m pip install -e . " + run_cmd = f"CC={env['CM_C_COMPILER_WITH_PATH']} CXX={env['CM_CXX_COMPILER_WITH_PATH']} USE_CUDA=OFF python -m pip install -e . " env['CM_RUN_CMD'] = run_cmd @@ -26,10 +27,12 @@ def preprocess(i): if not env.get('+ CXXFLAGS', []): env['+ CXXFLAGS'] = [] - env['+ CFLAGS'] += [ "-Wno-error=uninitialized", "-Wno-error=maybe-uninitialized", "-fno-strict-aliasing" ] - env['+ CXXFLAGS'] += [ "-Wno-error=uninitialized", "-Wno-error=maybe-uninitialized", "-fno-strict-aliasing" ] + env['+ CFLAGS'] += ["-Wno-error=uninitialized", + "-Wno-error=maybe-uninitialized", "-fno-strict-aliasing"] + env['+ CXXFLAGS'] += ["-Wno-error=uninitialized", + "-Wno-error=maybe-uninitialized", "-fno-strict-aliasing"] automation = i['automation'] recursion_spaces = i['recursion_spaces'] - return {'return':0} + return {'return': 0} diff --git a/script/install-pytorch-kineto-from-src/customize.py b/script/install-pytorch-kineto-from-src/customize.py index df2744ac4d..032a4f3bf2 100644 --- a/script/install-pytorch-kineto-from-src/customize.py +++ b/script/install-pytorch-kineto-from-src/customize.py @@ -1,12 +1,13 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] @@ -14,4 +15,4 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] - return {'return':0} + return {'return': 0} diff --git a/script/install-rapidjson-from-src/customize.py b/script/install-rapidjson-from-src/customize.py index 895731add7..6d5e7f9db3 100644 --- a/script/install-rapidjson-from-src/customize.py +++ b/script/install-rapidjson-from-src/customize.py @@ -1,12 +1,13 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] @@ -14,7 +15,8 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] - return {'return':0} + return {'return': 0} + def postprocess(i): - return {'return':0} + return {'return': 0} diff --git a/script/install-rocm/customize.py b/script/install-rocm/customize.py index 3ae7daafb6..2762f81389 100644 --- a/script/install-rocm/customize.py +++ b/script/install-rocm/customize.py @@ -1,12 +1,14 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} + + return {'return': 0} - return {'return':0} def postprocess(i): @@ -14,6 +16,6 @@ def postprocess(i): installed_path = "/opt/rocm/bin" env['CM_ROCM_INSTALLED_PATH'] = installed_path env['CM_ROCM_BIN_WITH_PATH'] = os.path.join(installed_path, "rocminfo") - env['+PATH'] = [ installed_path ] + env['+PATH'] = [installed_path] - return {'return':0} + return {'return': 0} diff --git a/script/install-tensorflow-for-c/customize.py b/script/install-tensorflow-for-c/customize.py index a0a8a42148..d950482c4e 100644 --- a/script/install-tensorflow-for-c/customize.py +++ b/script/install-tensorflow-for-c/customize.py @@ -1,27 +1,32 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} + + return {'return': 0} - return {'return':0} def postprocess(i): env = i['env'] - for key in ['+C_INCLUDE_PATH', '+CPLUS_INCLUDE_PATH', '+LD_LIBRARY_PATH', '+DYLD_FALLBACK_LIBRARY_PATH']: -# 20221024: we save and restore env in the main script and can clean env here for determinism -# if key not in env: + for key in ['+C_INCLUDE_PATH', '+CPLUS_INCLUDE_PATH', + '+LD_LIBRARY_PATH', '+DYLD_FALLBACK_LIBRARY_PATH']: + # 20221024: we save and restore env in the main script and can clean env here for determinism + # if key not in env: env[key] = [] - env['+C_INCLUDE_PATH'].append(os.path.join(os.getcwd(), 'install', 'include')) - env['+CPLUS_INCLUDE_PATH'].append(os.path.join(os.getcwd(), 'install', 'include')) + env['+C_INCLUDE_PATH'].append(os.path.join(os.getcwd(), + 'install', 'include')) + env['+CPLUS_INCLUDE_PATH'].append(os.path.join(os.getcwd(), + 'install', 'include')) lib_path = os.path.join(os.getcwd(), 'install', 'lib') env['+LD_LIBRARY_PATH'].append(lib_path) env['+DYLD_FALLBACK_LIBRARY_PATH'].append(lib_path) - return {'return':0} + return {'return': 0} diff --git a/script/install-tensorflow-from-src/customize.py b/script/install-tensorflow-from-src/customize.py index 3ddee9a900..f591d55227 100644 --- a/script/install-tensorflow-from-src/customize.py +++ b/script/install-tensorflow-from-src/customize.py @@ -1,34 +1,61 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] env['CC'] = env['CM_C_COMPILER_WITH_PATH'] - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - for key in ['+C_INCLUDE_PATH', '+CPLUS_INCLUDE_PATH', '+LD_LIBRARY_PATH', '+DYLD_FALLBACK_LIBRARY_PATH']: -# 20221024: we save and restore env in the main script and can clean env here for determinism -# if key not in env: + for key in ['+C_INCLUDE_PATH', '+CPLUS_INCLUDE_PATH', + '+LD_LIBRARY_PATH', '+DYLD_FALLBACK_LIBRARY_PATH']: + # 20221024: we save and restore env in the main script and can clean env here for determinism + # if key not in env: env[key] = [] bazel_install_root = os.path.join(os.getcwd(), "src", "bazel-out") bazel_install_bin = os.path.join(os.getcwd(), "src", "bazel-bin") inc_paths = [] inc_paths.append(os.path.join(os.getcwd(), "src")) inc_paths.append(bazel_install_bin) - inc_paths.append(os.path.join(bazel_install_bin, "external", "flatbuffers", "_virtual_includes", "flatbuffers")) - inc_paths.append(os.path.join(bazel_install_bin, "external", "FP16", "_virtual_includes", "FP16")) - inc_paths.append(os.path.join(bazel_install_bin, "external", "pthreadpool", "_virtual_includes", "pthreadpool")) - inc_paths.append(os.path.join(bazel_install_bin, "external", "cpuinfo", "_virtual_includes", "cpuinfo")) + inc_paths.append( + os.path.join( + bazel_install_bin, + "external", + "flatbuffers", + "_virtual_includes", + "flatbuffers")) + inc_paths.append( + os.path.join( + bazel_install_bin, + "external", + "FP16", + "_virtual_includes", + "FP16")) + inc_paths.append( + os.path.join( + bazel_install_bin, + "external", + "pthreadpool", + "_virtual_includes", + "pthreadpool")) + inc_paths.append( + os.path.join( + bazel_install_bin, + "external", + "cpuinfo", + "_virtual_includes", + "cpuinfo")) env['+C_INCLUDE_PATH'] = inc_paths env['+CPLUS_INCLUDE_PATH'] = inc_paths @@ -41,4 +68,4 @@ def postprocess(i): env['+LD_LIBRARY_PATH'].append(lib_path) env['+DYLD_FALLBACK_LIBRARY_PATH'].append(lib_path) - return {'return':0} + return {'return': 0} diff --git a/script/install-terraform-from-src/customize.py b/script/install-terraform-from-src/customize.py index 84fccf236e..fb46c1192f 100644 --- a/script/install-terraform-from-src/customize.py +++ b/script/install-terraform-from-src/customize.py @@ -1,19 +1,22 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} + + return {'return': 0} - return {'return':0} def postprocess(i): env = i['env'] installed_path = os.path.join(os.getcwd(), 'bin') env['CM_TERRAFORM_INSTALLED_PATH'] = installed_path - env['CM_TERRAFORM_BIN_WITH_PATH'] = os.path.join(installed_path, "terraform") - env['+PATH'] = [ installed_path ] + env['CM_TERRAFORM_BIN_WITH_PATH'] = os.path.join( + installed_path, "terraform") + env['+PATH'] = [installed_path] - return {'return':0} + return {'return': 0} diff --git a/script/install-tflite-from-src/customize.py b/script/install-tflite-from-src/customize.py index ab0055816d..b230459384 100644 --- a/script/install-tflite-from-src/customize.py +++ b/script/install-tflite-from-src/customize.py @@ -1,20 +1,23 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} + + return {'return': 0} - return {'return':0} def postprocess(i): env = i['env'] - for key in ['+C_INCLUDE_PATH', '+CPLUS_INCLUDE_PATH', '+LD_LIBRARY_PATH', '+DYLD_FALLBACK_LIBRARY_PATH']: -# 20221024: we save and restore env in the main script and can clean env here for determinism -# if key not in env: + for key in ['+C_INCLUDE_PATH', '+CPLUS_INCLUDE_PATH', + '+LD_LIBRARY_PATH', '+DYLD_FALLBACK_LIBRARY_PATH']: + # 20221024: we save and restore env in the main script and can clean env here for determinism + # if key not in env: env[key] = [] env['+C_INCLUDE_PATH'].append(os.path.join(os.getcwd(), 'src')) @@ -24,4 +27,4 @@ def postprocess(i): env['+LD_LIBRARY_PATH'].append(lib_path) env['+DYLD_FALLBACK_LIBRARY_PATH'].append(lib_path) - return {'return':0} + return {'return': 0} diff --git a/script/install-torchvision-from-src/customize.py b/script/install-torchvision-from-src/customize.py index 895731add7..6d5e7f9db3 100644 --- a/script/install-torchvision-from-src/customize.py +++ b/script/install-torchvision-from-src/customize.py @@ -1,12 +1,13 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] @@ -14,7 +15,8 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] - return {'return':0} + return {'return': 0} + def postprocess(i): - return {'return':0} + return {'return': 0} diff --git a/script/install-tpp-pytorch-extension/customize.py b/script/install-tpp-pytorch-extension/customize.py index 35ee0e05a6..6ee676b687 100644 --- a/script/install-tpp-pytorch-extension/customize.py +++ b/script/install-tpp-pytorch-extension/customize.py @@ -1,23 +1,25 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] env['TPP_PEX_DIR'] = env['CM_TPP_PEX_SRC_REPO_PATH'] env['DNNL_GRAPH_BUILD_COMPILER_BACKEND'] = 1 env['USE_LLVM'] = env['CM_LLVM_INSTALLED_PATH'] - env['LLVM_DIR'] = os.path.join(env['CM_LLVM_INSTALLED_PATH'], "lib", "cmake", "llvm") + env['LLVM_DIR'] = os.path.join( + env['CM_LLVM_INSTALLED_PATH'], "lib", "cmake", "llvm") - run_cmd="python setup.py clean && python setup.py install" + run_cmd = "python setup.py clean && python setup.py install" env['CM_RUN_DIR'] = env['TPP_PEX_DIR'] env['CM_RUN_CMD'] = run_cmd - return {'return':0} + return {'return': 0} diff --git a/script/install-transformers-from-src/customize.py b/script/install-transformers-from-src/customize.py index 29f69cfc69..c287139fe5 100644 --- a/script/install-transformers-from-src/customize.py +++ b/script/install-transformers-from-src/customize.py @@ -1,16 +1,17 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] - run_cmd="python setup.py install" + run_cmd = "python setup.py install" env['CM_RUN_CMD'] = run_cmd @@ -18,4 +19,4 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] - return {'return':0} + return {'return': 0} diff --git a/script/launch-benchmark/tests/debug.py b/script/launch-benchmark/tests/debug.py index 842003b2c6..7c8bab4b73 100644 --- a/script/launch-benchmark/tests/debug.py +++ b/script/launch-benchmark/tests/debug.py @@ -1,6 +1,6 @@ import cmind -r=cmind.access({'action':'gui', - 'automation':'script', - 'artifact':'launch benchmark'}) -print (r) +r = cmind.access({'action': 'gui', + 'automation': 'script', + 'artifact': 'launch benchmark'}) +print(r) diff --git a/script/plug-prebuilt-cudnn-to-cuda/customize.py b/script/plug-prebuilt-cudnn-to-cuda/customize.py index 912e9cba68..2beb23e1fa 100644 --- a/script/plug-prebuilt-cudnn-to-cuda/customize.py +++ b/script/plug-prebuilt-cudnn-to-cuda/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): recursion_spaces = i['recursion_spaces'] @@ -11,7 +12,7 @@ def preprocess(i): env = i['env'] - if str(env.get('CUDA_SKIP_SUDO','')).lower() == 'true': + if str(env.get('CUDA_SKIP_SUDO', '')).lower() == 'true': env['CM_SUDO'] = '' meta = i['meta'] @@ -21,7 +22,8 @@ def preprocess(i): supported_versions = list(meta['versions'].keys()) if version not in supported_versions: - return {'return': 1, 'error': "Only cuDNN versions {} are supported now".format(', '.join(supported_versions))} + return {'return': 1, 'error': "Only cuDNN versions {} are supported now".format( + ', '.join(supported_versions))} env['CM_CUDNN_VERSION'] = version @@ -39,12 +41,12 @@ def preprocess(i): cudnn_url = f'https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/{filename}' - print ('') - print (f'URL to download cuDNN: {cudnn_url}') + print('') + print(f'URL to download cuDNN: {cudnn_url}') env['CM_CUDNN_TAR_DIR'] = cudnn_dir env['CM_CUDNN_UNTAR_PATH'] = os.path.join(cur_dir, cudnn_dir) env['WGET_URL'] = cudnn_url env['CM_DOWNLOAD_CHECKSUM'] = cudnn_md5sum - return {'return':0} + return {'return': 0} diff --git a/script/plug-prebuilt-cusparselt-to-cuda/customize.py b/script/plug-prebuilt-cusparselt-to-cuda/customize.py index a2d7926d89..9262677055 100644 --- a/script/plug-prebuilt-cusparselt-to-cuda/customize.py +++ b/script/plug-prebuilt-cusparselt-to-cuda/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): recursion_spaces = i['recursion_spaces'] @@ -11,7 +12,7 @@ def preprocess(i): env = i['env'] - if str(env.get('CUDA_SKIP_SUDO','')).lower() == 'true': + if str(env.get('CUDA_SKIP_SUDO', '')).lower() == 'true': env['CM_SUDO'] = '' meta = i['meta'] @@ -21,7 +22,8 @@ def preprocess(i): supported_versions = list(meta['versions'].keys()) if version not in supported_versions: - return {'return': 1, 'error': "Only CUSPARSELT versions {} are supported now".format(', '.join(supported_versions))} + return {'return': 1, 'error': "Only CUSPARSELT versions {} are supported now".format( + ', '.join(supported_versions))} env['CM_CUSPARSELT_VERSION'] = version @@ -39,12 +41,12 @@ def preprocess(i): cusparselt_url = f'https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-x86_64/{filename}' - print ('') - print (f'URL to download CUSPARSELT: {cusparselt_url}') + print('') + print(f'URL to download CUSPARSELT: {cusparselt_url}') env['CM_CUSPARSELT_TAR_DIR'] = cusparselt_dir env['CM_CUSPARSELT_UNTAR_PATH'] = os.path.join(cur_dir, cusparselt_dir) env['WGET_URL'] = cusparselt_url env['CM_DOWNLOAD_CHECKSUM'] = cusparselt_md5sum - return {'return':0} + return {'return': 0} diff --git a/script/print-hello-world-py/app.py b/script/print-hello-world-py/app.py index 9dfd48b33a..12382ac80d 100644 --- a/script/print-hello-world-py/app.py +++ b/script/print-hello-world-py/app.py @@ -1,16 +1,20 @@ def main(): - print ('') + print('') # Import cmind to test break points import cmind.utils import os if os.environ.get('CM_TMP_DEBUG_UID', '') == 'f52670e5f3f345a2': - cmind.utils.debug_here(__file__, port=5678, text='Debugging main.py!').breakpoint() + cmind.utils.debug_here( + __file__, + port=5678, + text='Debugging main.py!').breakpoint() - print ('HELLO WORLD from Python') + print('HELLO WORLD from Python') x = 1 - print (x) + print(x) + if __name__ == '__main__': main() diff --git a/script/print-hello-world-py/customize.py b/script/print-hello-world-py/customize.py index 8308693677..3ee63ecb18 100644 --- a/script/print-hello-world-py/customize.py +++ b/script/print-hello-world-py/customize.py @@ -2,13 +2,14 @@ import os + def preprocess(i): os_info = i['os_info'] env = i['env'] meta = i['meta'] - return {'return':0} + return {'return': 0} def postprocess(i): @@ -16,4 +17,4 @@ def postprocess(i): env = i['env'] state = i['state'] - return {'return':0} + return {'return': 0} diff --git a/script/process-ae-users/customize.py b/script/process-ae-users/customize.py index fef0a3ddfd..e0f6b1ef23 100644 --- a/script/process-ae-users/customize.py +++ b/script/process-ae-users/customize.py @@ -2,9 +2,10 @@ import cmind as cm import os + def preprocess(i): os_info = i['os_info'] env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/prune-bert-models/customize.py b/script/prune-bert-models/customize.py index 34e0810231..ae3a704e98 100644 --- a/script/prune-bert-models/customize.py +++ b/script/prune-bert-models/customize.py @@ -1,44 +1,49 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] env = i['env'] - ckpt_path = env.get('CM_BERT_PRUNE_CKPT_PATH','') + ckpt_path = env.get('CM_BERT_PRUNE_CKPT_PATH', '') if ckpt_path == '': p = env['CM_ML_MODEL_FILE_WITH_PATH'] x = os.listdir(p) for y in x: if y.startswith('models--'): - z = os.path.join(p,y) + z = os.path.join(p, y) if os.path.isdir(z): z1 = os.path.join(z, 'snapshots') if os.path.isdir(z1): z2 = os.listdir(z1) - if len(z2)>0: - ckpt_path=os.path.join(z1, z2[0]) + if len(z2) > 0: + ckpt_path = os.path.join(z1, z2[0]) env['CM_BERT_PRUNE_CKPT_PATH'] = ckpt_path - out_dir=env.get('CM_BERT_PRUNE_OUTPUT_DIR','') + out_dir = env.get('CM_BERT_PRUNE_OUTPUT_DIR', '') if out_dir == '': out_dir = os.path.join(os.getcwd(), 'pruned-model-output') env['CM_BERT_PRUNE_OUTPUT_DIR'] = out_dir - print ('') - print ('Local CM cache path to the updated BERT pruner src from NeurIPS 2022: ' + env['CM_GIT_REPO_BERT_PRUNER_NEURIPS_2022_CHECKOUT_PATH']) + print('') + print( + 'Local CM cache path to the updated BERT pruner src from NeurIPS 2022: ' + + env['CM_GIT_REPO_BERT_PRUNER_NEURIPS_2022_CHECKOUT_PATH']) - print ('') - for k in ["CM_ML_MODEL_FILE_WITH_PATH", "CM_BERT_PRUNE_CKPT_PATH", "CM_BERT_PRUNE_OUTPUT_DIR"]: - print ('ENV["{}"]: {}'.format(k, env[k])) + print('') + for k in ["CM_ML_MODEL_FILE_WITH_PATH", + "CM_BERT_PRUNE_CKPT_PATH", "CM_BERT_PRUNE_OUTPUT_DIR"]: + print('ENV["{}"]: {}'.format(k, env[k])) - print ('') + print('') return {'return': 0} + def postprocess(i): env = i['env'] diff --git a/script/pull-git-repo/customize.py b/script/pull-git-repo/customize.py index 021d42465e..55a581bb5c 100644 --- a/script/pull-git-repo/customize.py +++ b/script/pull-git-repo/customize.py @@ -2,22 +2,23 @@ import os import shutil + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] meta = i['meta'] if 'CM_GIT_CHECKOUT_PATH' not in env: - return {'return':1, 'error': 'CM_GIT_CHECKOUT_PATH is not set'} + return {'return': 1, 'error': 'CM_GIT_CHECKOUT_PATH is not set'} env['CM_GIT_PULL_CMD'] = "git pull --rebase" - return {'return':0} + return {'return': 0} def postprocess(i): @@ -25,4 +26,4 @@ def postprocess(i): env = i['env'] state = i['state'] - return {'return':0} + return {'return': 0} diff --git a/script/push-csv-to-spreadsheet/customize.py b/script/push-csv-to-spreadsheet/customize.py index e80f262666..213925e2f0 100644 --- a/script/push-csv-to-spreadsheet/customize.py +++ b/script/push-csv-to-spreadsheet/customize.py @@ -2,6 +2,7 @@ import cmind as cm import os + def preprocess(i): os_info = i['os_info'] @@ -9,7 +10,8 @@ def preprocess(i): meta = i['meta'] automation = i['automation'] - return {'return':0} + return {'return': 0} + def postprocess(i): - return {'return':0} + return {'return': 0} diff --git a/script/push-csv-to-spreadsheet/google_api.py b/script/push-csv-to-spreadsheet/google_api.py index d1e7643aa4..24926daed0 100644 --- a/script/push-csv-to-spreadsheet/google_api.py +++ b/script/push-csv-to-spreadsheet/google_api.py @@ -45,7 +45,12 @@ def main(): f = open(csv_file, "r") values = [r for r in csv.reader(f)] - request = service.spreadsheets().values().update(spreadsheetId=DOCUMENT_ID, range=sheet_name, valueInputOption="USER_ENTERED", body={"values": values}).execute() + request = service.spreadsheets().values().update( + spreadsheetId=DOCUMENT_ID, + range=sheet_name, + valueInputOption="USER_ENTERED", + body={ + "values": values}).execute() except HttpError as err: print(err) diff --git a/script/push-mlperf-inference-results-to-github/customize.py b/script/push-mlperf-inference-results-to-github/customize.py index dcdc3c0bfe..f1cfe1ebab 100644 --- a/script/push-mlperf-inference-results-to-github/customize.py +++ b/script/push-mlperf-inference-results-to-github/customize.py @@ -2,6 +2,7 @@ import cmind as cm import os + def preprocess(i): os_info = i['os_info'] @@ -19,17 +20,20 @@ def preprocess(i): else: extra_tags_string = "" - r = automation.update_deps({'deps':meta['prehook_deps'], - 'update_deps':{ - 'get-git-repo':{ - 'tags':"_repo."+repo+extra_tags_string - } - } - }) - if r['return']>0: return r - env['CM_MLPERF_RESULTS_REPO_COMMIT_MESSAGE'] = env.get('CM_MLPERF_RESULTS_REPO_COMMIT_MESSAGE', 'Added new results') + r = automation.update_deps({'deps': meta['prehook_deps'], + 'update_deps': { + 'get-git-repo': { + 'tags': "_repo." + repo + extra_tags_string + } + } + }) + if r['return'] > 0: + return r + env['CM_MLPERF_RESULTS_REPO_COMMIT_MESSAGE'] = env.get( + 'CM_MLPERF_RESULTS_REPO_COMMIT_MESSAGE', 'Added new results') + + return {'return': 0} - return {'return':0} def postprocess(i): - return {'return':0} + return {'return': 0} diff --git a/script/reproduce-ieee-acm-micro2023-paper-22/customize.py b/script/reproduce-ieee-acm-micro2023-paper-22/customize.py index d12f9b3e1d..273999d460 100644 --- a/script/reproduce-ieee-acm-micro2023-paper-22/customize.py +++ b/script/reproduce-ieee-acm-micro2023-paper-22/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,10 +14,11 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/reproduce-ieee-acm-micro2023-paper-28/customize.py b/script/reproduce-ieee-acm-micro2023-paper-28/customize.py index d12f9b3e1d..273999d460 100644 --- a/script/reproduce-ieee-acm-micro2023-paper-28/customize.py +++ b/script/reproduce-ieee-acm-micro2023-paper-28/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,10 +14,11 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/reproduce-ieee-acm-micro2023-paper-33/customize.py b/script/reproduce-ieee-acm-micro2023-paper-33/customize.py index d12f9b3e1d..273999d460 100644 --- a/script/reproduce-ieee-acm-micro2023-paper-33/customize.py +++ b/script/reproduce-ieee-acm-micro2023-paper-33/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,10 +14,11 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/reproduce-ieee-acm-micro2023-paper-5/customize.py b/script/reproduce-ieee-acm-micro2023-paper-5/customize.py index d12f9b3e1d..273999d460 100644 --- a/script/reproduce-ieee-acm-micro2023-paper-5/customize.py +++ b/script/reproduce-ieee-acm-micro2023-paper-5/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,10 +14,11 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/reproduce-ieee-acm-micro2023-paper-5/main.py b/script/reproduce-ieee-acm-micro2023-paper-5/main.py index d851f1450f..caa499bf08 100644 --- a/script/reproduce-ieee-acm-micro2023-paper-5/main.py +++ b/script/reproduce-ieee-acm-micro2023-paper-5/main.py @@ -2,9 +2,9 @@ if __name__ == "__main__": - print ('') - print ('Main script:') - print ('Experiment: {}'.format(os.environ.get('CM_EXPERIMENT',''))) - print ('') + print('') + print('Main script:') + print('Experiment: {}'.format(os.environ.get('CM_EXPERIMENT', ''))) + print('') exit(0) diff --git a/script/reproduce-ieee-acm-micro2023-paper-8/customize.py b/script/reproduce-ieee-acm-micro2023-paper-8/customize.py index d12f9b3e1d..273999d460 100644 --- a/script/reproduce-ieee-acm-micro2023-paper-8/customize.py +++ b/script/reproduce-ieee-acm-micro2023-paper-8/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,10 +14,11 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/reproduce-ieee-acm-micro2023-paper-8/main.py b/script/reproduce-ieee-acm-micro2023-paper-8/main.py index d851f1450f..caa499bf08 100644 --- a/script/reproduce-ieee-acm-micro2023-paper-8/main.py +++ b/script/reproduce-ieee-acm-micro2023-paper-8/main.py @@ -2,9 +2,9 @@ if __name__ == "__main__": - print ('') - print ('Main script:') - print ('Experiment: {}'.format(os.environ.get('CM_EXPERIMENT',''))) - print ('') + print('') + print('Main script:') + print('Experiment: {}'.format(os.environ.get('CM_EXPERIMENT', ''))) + print('') exit(0) diff --git a/script/reproduce-ieee-acm-micro2023-paper-85/customize.py b/script/reproduce-ieee-acm-micro2023-paper-85/customize.py index d12f9b3e1d..273999d460 100644 --- a/script/reproduce-ieee-acm-micro2023-paper-85/customize.py +++ b/script/reproduce-ieee-acm-micro2023-paper-85/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,10 +14,11 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/reproduce-ieee-acm-micro2023-paper-96/customize.py b/script/reproduce-ieee-acm-micro2023-paper-96/customize.py index d12f9b3e1d..273999d460 100644 --- a/script/reproduce-ieee-acm-micro2023-paper-96/customize.py +++ b/script/reproduce-ieee-acm-micro2023-paper-96/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,10 +14,11 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/reproduce-ieee-acm-micro2023-paper-96/main.py b/script/reproduce-ieee-acm-micro2023-paper-96/main.py index d851f1450f..caa499bf08 100644 --- a/script/reproduce-ieee-acm-micro2023-paper-96/main.py +++ b/script/reproduce-ieee-acm-micro2023-paper-96/main.py @@ -2,9 +2,9 @@ if __name__ == "__main__": - print ('') - print ('Main script:') - print ('Experiment: {}'.format(os.environ.get('CM_EXPERIMENT',''))) - print ('') + print('') + print('Main script:') + print('Experiment: {}'.format(os.environ.get('CM_EXPERIMENT', ''))) + print('') exit(0) diff --git a/script/reproduce-ipol-paper-2022-439/customize.py b/script/reproduce-ipol-paper-2022-439/customize.py index 6b57ab932f..fca74f12fc 100644 --- a/script/reproduce-ipol-paper-2022-439/customize.py +++ b/script/reproduce-ipol-paper-2022-439/customize.py @@ -1,34 +1,40 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] env = i['env'] # Check if input files are empty and add files - input_file_1 = env.get('CM_INPUT_1','') - if input_file_1 == '': input_file_1 = 'ipol-paper-2024-439-sample-image-1.png' + input_file_1 = env.get('CM_INPUT_1', '') + if input_file_1 == '': + input_file_1 = 'ipol-paper-2024-439-sample-image-1.png' if not os.path.isfile(input_file_1): - return {'return':1, 'error':'input file 1 "{}" not found'.format(input_file_1)} + return {'return': 1, + 'error': 'input file 1 "{}" not found'.format(input_file_1)} - env['CM_INPUT_1']=os.path.abspath(input_file_1) + env['CM_INPUT_1'] = os.path.abspath(input_file_1) - input_file_2 = env.get('CM_INPUT_2','') - if input_file_2 == '': input_file_2 = 'ipol-paper-2024-439-sample-image-2.png' + input_file_2 = env.get('CM_INPUT_2', '') + if input_file_2 == '': + input_file_2 = 'ipol-paper-2024-439-sample-image-2.png' if not os.path.isfile(input_file_2): - return {'return':1, 'error':'input file 2 "{}" not found'.format(input_file_2)} + return {'return': 1, + 'error': 'input file 2 "{}" not found'.format(input_file_2)} + + env['CM_INPUT_2'] = os.path.abspath(input_file_2) - env['CM_INPUT_2']=os.path.abspath(input_file_2) + return {'return': 0} - return {'return':0} def postprocess(i): - print ('') - print ('Please check "diff.png"') - print ('') + print('') + print('Please check "diff.png"') + print('') - return {'return':0} + return {'return': 0} diff --git a/script/reproduce-mlperf-octoml-tinyml-results/customize.py b/script/reproduce-mlperf-octoml-tinyml-results/customize.py index f63d95abaa..7e9eac6901 100644 --- a/script/reproduce-mlperf-octoml-tinyml-results/customize.py +++ b/script/reproduce-mlperf-octoml-tinyml-results/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -12,12 +13,13 @@ def preprocess(i): env['CM_TINY_MODEL'] = 'ic' if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env['+C_INCLUDE_PATH'] = [] - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/reproduce-mlperf-training-nvidia/customize.py b/script/reproduce-mlperf-training-nvidia/customize.py index 25e9929d82..cdc98cc7c8 100644 --- a/script/reproduce-mlperf-training-nvidia/customize.py +++ b/script/reproduce-mlperf-training-nvidia/customize.py @@ -2,17 +2,19 @@ import os import shutil + def preprocess(i): os_info = i['os_info'] if os_info['platform'] == 'windows': - return {'return':1, 'error': 'Windows is not supported in this script yet'} + return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] conf = env.get('CM_MLPERF_NVIDIA_TRAINING_SYSTEM_CONF_NAME', '') if conf == "": - return {'return':1, 'error': 'Please provide --system_conf_name='} + return {'return': 1, + 'error': 'Please provide --system_conf_name='} if not conf.endswith(".sh"): conf = conf + ".sh" @@ -23,10 +25,11 @@ def preprocess(i): env['CONFIG_FILE'] = conf # print(env) - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/run-mlperf-inference-mobilenet-models/customize.py b/script/run-mlperf-inference-mobilenet-models/customize.py index ace19a6fd8..b6aabca6b6 100644 --- a/script/run-mlperf-inference-mobilenet-models/customize.py +++ b/script/run-mlperf-inference-mobilenet-models/customize.py @@ -24,29 +24,29 @@ def preprocess(i): models_all = { "mobilenet": { "v1": { - "multiplier": [ "multiplier-1.0", "multiplier-0.75", "multiplier-0.5", "multiplier-0.25" ], - "resolution": [ "resolution-224", "resolution-192", "resolution-160", "resolution-128" ], + "multiplier": ["multiplier-1.0", "multiplier-0.75", "multiplier-0.5", "multiplier-0.25"], + "resolution": ["resolution-224", "resolution-192", "resolution-160", "resolution-128"], "kind": [""] - }, + }, "v2": { - "multiplier": [ "multiplier-1.0", "multiplier-0.75", "multiplier-0.5", "multiplier-0.35" ], - "resolution": [ "resolution-224", "resolution-192", "resolution-160", "resolution-128" ], + "multiplier": ["multiplier-1.0", "multiplier-0.75", "multiplier-0.5", "multiplier-0.35"], + "resolution": ["resolution-224", "resolution-192", "resolution-160", "resolution-128"], "kind": [""] - }, + }, "v3": { "multiplier": [""], "resolution": [""], - "kind": [ "large", "large-minimalistic", "small", "small-minimalistic" ] - } - }, + "kind": ["large", "large-minimalistic", "small", "small-minimalistic"] + } + }, "efficientnet": { "": { "multiplier": [""], "resolution": [""], - "kind": [ "lite0", "lite1", "lite2", "lite3", "lite4" ] - } + "kind": ["lite0", "lite1", "lite2", "lite3", "lite4"] } } + } models = {} if env.get('CM_MLPERF_RUN_MOBILENET_V1', '') == "yes": @@ -71,41 +71,41 @@ def preprocess(i): for version in models[t1]: variation_list = [] if version.strip(): - variation_list.append("_"+version) + variation_list.append("_" + version) variation_list_saved = variation_list.copy() for k1 in models[t1][version]["multiplier"]: variation_list = variation_list_saved.copy() if k1.strip(): - variation_list.append("_"+k1) + variation_list.append("_" + k1) variation_list_saved_2 = variation_list.copy() for k2 in models[t1][version]["resolution"]: variation_list = variation_list_saved_2.copy() if k2.strip(): - variation_list.append("_"+k2) + variation_list.append("_" + k2) variation_list_saved_3 = variation_list.copy() for k3 in models[t1][version]["kind"]: variation_list = variation_list_saved_3.copy() if k3.strip(): - variation_list.append("_"+k3) + variation_list.append("_" + k3) variation_strings[t1].append(",".join(variation_list)) - if env.get('CM_MLPERF_SUBMISSION_MODE','') == "yes": - var="_submission" - execution_mode="valid" - elif env.get('CM_MLPERF_ACCURACY_MODE','') == "yes" and env.get('CM_MLPERF_PERFORMANCE_MODE','') == "yes": - var="_full,_performance-and-accuracy" - execution_mode="valid" - elif env.get('CM_MLPERF_ACCURACY_MODE','') == "yes": - var="_full,_accuracy-only" - execution_mode="valid" - elif env.get('CM_MLPERF_PERFORMANCE_MODE','') == "yes": - var="_full,_performance-only" - execution_mode="valid" + if env.get('CM_MLPERF_SUBMISSION_MODE', '') == "yes": + var = "_submission" + execution_mode = "valid" + elif env.get('CM_MLPERF_ACCURACY_MODE', '') == "yes" and env.get('CM_MLPERF_PERFORMANCE_MODE', '') == "yes": + var = "_full,_performance-and-accuracy" + execution_mode = "valid" + elif env.get('CM_MLPERF_ACCURACY_MODE', '') == "yes": + var = "_full,_accuracy-only" + execution_mode = "valid" + elif env.get('CM_MLPERF_PERFORMANCE_MODE', '') == "yes": + var = "_full,_performance-only" + execution_mode = "valid" else: - var="_find-performance" - execution_mode="test" + var = "_find-performance" + execution_mode = "test" - precisions = [ ] + precisions = [] if env.get('CM_MLPERF_RUN_FP32', '') == "yes": precisions.append("fp32") if env.get('CM_MLPERF_RUN_INT8', '') == "yes": @@ -156,10 +156,12 @@ def preprocess(i): } } if add_deps_recursive: - cm_input['add_deps_recursive'] = add_deps_recursive #script automation will merge adr and add_deps_recursive + # script automation will merge adr and add_deps_recursive + cm_input['add_deps_recursive'] = add_deps_recursive if adr: - utils.merge_dicts({'dict1':cm_input['adr'], 'dict2':adr, 'append_lists':True, 'append_unique':True}) + utils.merge_dicts( + {'dict1': cm_input['adr'], 'dict2': adr, 'append_lists': True, 'append_unique': True}) if env.get('CM_MLPERF_INFERENCE_RESULTS_DIR', '') != '': cm_input['results_dir'] = env['CM_MLPERF_INFERENCE_RESULTS_DIR'] @@ -167,20 +169,21 @@ def preprocess(i): if env.get('CM_MLPERF_INFERENCE_SUBMISSION_DIR', '') != '': cm_input['submission_dir'] = env['CM_MLPERF_INFERENCE_SUBMISSION_DIR'] - if env.get('CM_MLPERF_FIND_PERFORMANCE_MODE','') == "yes" and env.get('CM_MLPERF_NO_RERUN','') != 'yes': + if env.get('CM_MLPERF_FIND_PERFORMANCE_MODE', '') == "yes" and env.get( + 'CM_MLPERF_NO_RERUN', '') != 'yes': cm_input['rerun'] = True - if env.get('CM_MLPERF_POWER','') == "yes": + if env.get('CM_MLPERF_POWER', '') == "yes": cm_input['power'] = 'yes' - if env.get('CM_MLPERF_ACCURACY_MODE','') == "yes": + if env.get('CM_MLPERF_ACCURACY_MODE', '') == "yes": cm_input['mode'] = 'accuracy' print(cm_input) r = cmind.access(cm_input) if r['return'] > 0: return r - if env.get('CM_MLPERF_PERFORMANCE_MODE','') == "yes": + if env.get('CM_MLPERF_PERFORMANCE_MODE', '') == "yes": cm_input['mode'] = 'performance' print(cm_input) @@ -189,21 +192,22 @@ def preprocess(i): return r if env.get('CM_TEST_ONE_RUN', '') == "yes": - return {'return':0} + return {'return': 0} clean_input = { - 'action': 'rm', - 'automation': 'cache', - 'tags': 'get,preprocessed,dataset,_for.mobilenet', + 'action': 'rm', + 'automation': 'cache', + 'tags': 'get,preprocessed,dataset,_for.mobilenet', 'quiet': True, 'v': verbose, 'f': 'True' - } + } r = cmind.access(clean_input) - #if r['return'] > 0: + # if r['return'] > 0: # return r - return {'return':0} + return {'return': 0} + def postprocess(i): - return {'return':0} + return {'return': 0} diff --git a/script/run-mlperf-inference-submission-checker/code.py b/script/run-mlperf-inference-submission-checker/code.py index 892d16be33..85fce01bc5 100644 --- a/script/run-mlperf-inference-submission-checker/code.py +++ b/script/run-mlperf-inference-submission-checker/code.py @@ -3,25 +3,27 @@ import os import pandas + def main(): - print ('=========================================================') + print('=========================================================') - print ('Searching for summary.csv ...') + print('Searching for summary.csv ...') if os.path.isfile('summary.csv'): - print ('Converting to json ...') + print('Converting to json ...') import pandas df = pandas.read_csv('summary.csv').T - print ('') - print (df) - print ('') + print('') + print(df) + print('') df.to_json('summary.json', orient='columns', indent=4) - print ('=========================================================') + print('=========================================================') + if __name__ == '__main__': main() diff --git a/script/run-mlperf-power-server/customize.py b/script/run-mlperf-power-server/customize.py index 65c7830420..ea989bb401 100644 --- a/script/run-mlperf-power-server/customize.py +++ b/script/run-mlperf-power-server/customize.py @@ -3,15 +3,20 @@ import os import configparser + def preprocess(i): os_info = i['os_info'] env = i['env'] config = configparser.ConfigParser() - server_config_file = os.path.join(env['CM_MLPERF_POWER_SOURCE'], 'ptd_client_server', 'server.template.conf') + server_config_file = os.path.join( + env['CM_MLPERF_POWER_SOURCE'], + 'ptd_client_server', + 'server.template.conf') config.read(server_config_file) config['server']['ntpServer'] = env['CM_MLPERF_POWER_NTP_SERVER'] - config['server']['listen'] = env['CM_MLPERF_POWER_SERVER_ADDRESS'] + " " + env['CM_MLPERF_POWER_SERVER_PORT'] + config['server']['listen'] = env['CM_MLPERF_POWER_SERVER_ADDRESS'] + \ + " " + env['CM_MLPERF_POWER_SERVER_PORT'] config['ptd']['ptd'] = env['CM_MLPERF_PTD_PATH'] config['ptd']['interfaceFlag'] = env['CM_MLPERF_POWER_INTERFACE_FLAG'] config['ptd']['deviceType'] = env['CM_MLPERF_POWER_DEVICE_TYPE'] @@ -25,7 +30,10 @@ def preprocess(i): else: cmd_prefix = "sudo " - cmd = env['CM_PYTHON_BIN_WITH_PATH'] + ' ' + os.path.join(env['CM_MLPERF_POWER_SOURCE'], 'ptd_client_server', 'server.py') +' -c power-server.conf' + cmd = env['CM_PYTHON_BIN_WITH_PATH'] + ' ' + os.path.join( + env['CM_MLPERF_POWER_SOURCE'], + 'ptd_client_server', + 'server.py') + ' -c power-server.conf' if env.get('CM_MLPERF_POWER_SERVER_USE_SCREEN', 'no') == 'yes': cmd = cmd_prefix + ' screen -d -m ' + cmd + ' ' else: @@ -33,7 +41,8 @@ def preprocess(i): env['RUN_CMD'] = cmd - return {'return':0} + return {'return': 0} + def postprocess(i): - return {'return':0} + return {'return': 0} diff --git a/script/run-terraform/customize.py b/script/run-terraform/customize.py index eeddbff60d..b1939f15c1 100644 --- a/script/run-terraform/customize.py +++ b/script/run-terraform/customize.py @@ -4,12 +4,15 @@ import shutil import json + def preprocess(i): os_info = i['os_info'] env = i['env'] script_dir = i['run_script_input']['path'] - config_dir = os.path.join(script_dir, env.get('CM_TERRAFORM_CONFIG_DIR_NAME', '')) + config_dir = os.path.join( + script_dir, env.get( + 'CM_TERRAFORM_CONFIG_DIR_NAME', '')) env['CM_TERRAFORM_CONFIG_DIR'] = config_dir cache_dir = os.getcwd() @@ -20,6 +23,7 @@ def preprocess(i): return {'return': 0} + def postprocess(i): env = i['env'] if env.get('CM_DESTROY_TERRAFORM'): @@ -47,7 +51,7 @@ def postprocess(i): 'action': 'run', 'tags': 'remote,run,ssh', 'env': { - }, + }, 'host': public_ip, 'user': user, 'skip_host_verify': True, @@ -62,18 +66,18 @@ def postprocess(i): "source ~/.profile", "cm pull repo ctuning@mlcommons-ck", "cm run script --tags=get,sys-utils-cm" - ] - } + ] + } if env.get('CM_TERRAFORM_RUN_COMMANDS'): run_cmds = env.get('CM_TERRAFORM_RUN_COMMANDS') for cmd in run_cmds: - cmd=cmd.replace(":", "=") - cmd=cmd.replace(";;", ",") + cmd = cmd.replace(":", "=") + cmd = cmd.replace(";;", ",") run_input['run_cmds'].append(cmd) r = cm.access(run_input) if r['return'] > 0: return r - #print(r) + # print(r) print_attr(instance_attributes, "id") print_attr(instance_attributes, "instance_type") print_attr(instance_attributes, "public_ip") @@ -82,6 +86,7 @@ def postprocess(i): return {'return': 0} + def print_attr(instance_attributes, key): if key in instance_attributes: print(key.upper() + ": " + str(instance_attributes[key])) diff --git a/script/save-mlperf-inference-implementation-state/customize.py b/script/save-mlperf-inference-implementation-state/customize.py index be3be96798..07e7169559 100644 --- a/script/save-mlperf-inference-implementation-state/customize.py +++ b/script/save-mlperf-inference-implementation-state/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -14,27 +15,30 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - if not state.get('mlperf-inference-implementation'): #No state information. Just returning + if not state.get( + 'mlperf-inference-implementation'): # No state information. Just returning return {'return': 0} if env.get('CM_MLPERF_README', "") == "yes": import cmind as cm inp = i['input'] - script_tags = state['mlperf-inference-implementation'].get('script_tags', '') - script_adr = state['mlperf-inference-implementation'].get('script_adr', {}) + script_tags = state['mlperf-inference-implementation'].get( + 'script_tags', '') + script_adr = state['mlperf-inference-implementation'].get( + 'script_adr', {}) if script_tags != '': cm_input = {'action': 'run', - 'automation': 'script', - 'tags': script_tags, - 'adr': script_adr, - 'env': env, - 'print_deps': True, - 'quiet': True, - 'silent': True, - 'fake_run': True - } + 'automation': 'script', + 'tags': script_tags, + 'adr': script_adr, + 'env': env, + 'print_deps': True, + 'quiet': True, + 'silent': True, + 'fake_run': True + } r = cm.access(cm_input) if r['return'] > 0: @@ -54,10 +58,11 @@ def preprocess(i): state['mlperf-inference-implementation']['version_info'] = version_info - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/set-device-settings-qaic/customize.py b/script/set-device-settings-qaic/customize.py index 48d065c84e..3f3e179f71 100644 --- a/script/set-device-settings-qaic/customize.py +++ b/script/set-device-settings-qaic/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -21,19 +22,23 @@ def preprocess(i): ecc_template['request'].append({}) ecc_template['request'][0]['qid'] = device ecc_template['request'][0]['dev_config'] = {} - ecc_template['request'][0]['dev_config']['update_ras_ecc_config_request'] = {} - ecc_template['request'][0]['dev_config']['update_ras_ecc_config_request']['ras_ecc'] = [] - ecc_template['request'][0]['dev_config']['update_ras_ecc_config_request']['ras_ecc'].append("RAS_DDR_ECC") - with open("request_"+device+".json", "w") as f: + ecc_template['request'][0]['dev_config']['update_ras_ecc_config_request'] = { + } + ecc_template['request'][0]['dev_config']['update_ras_ecc_config_request']['ras_ecc'] = [ + ] + ecc_template['request'][0]['dev_config']['update_ras_ecc_config_request']['ras_ecc'].append( + "RAS_DDR_ECC") + with open("request_" + device + ".json", "w") as f: f.write(json.dumps(ecc_template)) if env.get('CM_QAIC_VC', '') != '': env['CM_QAIC_VC_HEX'] = hex(int(env['CM_QAIC_VC'])) - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/set-echo-off-win/customize.py b/script/set-echo-off-win/customize.py index ef9ba8b8dd..287737629c 100644 --- a/script/set-echo-off-win/customize.py +++ b/script/set-echo-off-win/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -10,16 +11,16 @@ def preprocess(i): # If windows, download here otherwise use run.sh if os_info['platform'] == 'windows': - script_prefix = state.get('script_prefix',[]) + script_prefix = state.get('script_prefix', []) - s='@echo off' + s = '@echo off' if s not in script_prefix: script_prefix.insert(0, s) state['script_prefix'] = script_prefix # Test to skip next dependency - #env = i['env'] - #env['CM_SKIP_SYS_UTILS'] = 'YES' + # env = i['env'] + # env['CM_SKIP_SYS_UTILS'] = 'YES' - return {'return':0} + return {'return': 0} diff --git a/script/set-performance-mode/customize.py b/script/set-performance-mode/customize.py index fca19d718c..288ebc9c05 100644 --- a/script/set-performance-mode/customize.py +++ b/script/set-performance-mode/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -14,10 +15,11 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/set-sqlite-dir/customize.py b/script/set-sqlite-dir/customize.py index 638e68ff0c..21ab396e48 100644 --- a/script/set-sqlite-dir/customize.py +++ b/script/set-sqlite-dir/customize.py @@ -1,9 +1,10 @@ import os + def postprocess(i): env = i['env'] env['CM_SQLITE_PATH'] = os.getcwd() - return {'return':0} + return {'return': 0} diff --git a/script/set-user-limits/customize.py b/script/set-user-limits/customize.py index 3b67e410b3..ff5767250e 100644 --- a/script/set-user-limits/customize.py +++ b/script/set-user-limits/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -20,10 +21,11 @@ def preprocess(i): env['CM_RUN_CMD'] = " && ".join(cmds) - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/tar-my-folder/customize.py b/script/tar-my-folder/customize.py index d405e76fdb..c1d8498b89 100644 --- a/script/tar-my-folder/customize.py +++ b/script/tar-my-folder/customize.py @@ -4,6 +4,7 @@ import subprocess from os.path import exists + def preprocess(i): os_info = i['os_info'] @@ -17,13 +18,15 @@ def preprocess(i): output_file = env.get("CM_TAR_OUTFILE", "") input_dirname = os.path.basename(input_dir) if output_file == "": - output_file = input_dirname+".tar.gz" + output_file = input_dirname + ".tar.gz" from pathlib import Path input_path = Path(input_dir) cd_dir = input_path.parent.absolute() - CMD = 'tar --directory '+str(cd_dir)+' -czf ' + os.path.join(output_dir, output_file) + ' ' + input_dirname + CMD = 'tar --directory ' + \ + str(cd_dir) + ' -czf ' + os.path.join(output_dir, + output_file) + ' ' + input_dirname print(CMD) ret = os.system(CMD) - print("Tar file "+os.path.join(output_dir, output_file)+ " created") + print("Tar file " + os.path.join(output_dir, output_file) + " created") - return {'return':ret} + return {'return': ret} diff --git a/script/test-cm-core/customize.py b/script/test-cm-core/customize.py index 831f0bdff0..c3289f3451 100644 --- a/script/test-cm-core/customize.py +++ b/script/test-cm-core/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,4 +14,4 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} diff --git a/script/test-cm-core/src/script/check.py b/script/test-cm-core/src/script/check.py index 7394406d8f..4883116c03 100644 --- a/script/test-cm-core/src/script/check.py +++ b/script/test-cm-core/src/script/check.py @@ -1,9 +1,11 @@ def check_return(r): if 'return' not in r: - raise Exception('CM access function should always return key \'return\'!') + raise Exception( + 'CM access function should always return key \'return\'!') if 'error' in r: raise Exception(r['error']) + def check_list(r, string, found=True): check_return(r) if 'list' not in r: diff --git a/script/test-cm-core/src/script/process_dockerfile.py b/script/test-cm-core/src/script/process_dockerfile.py index 62bf4d5f87..1d23ec6015 100644 --- a/script/test-cm-core/src/script/process_dockerfile.py +++ b/script/test-cm-core/src/script/process_dockerfile.py @@ -5,7 +5,7 @@ import json import yaml -files=sys.argv[1:] +files = sys.argv[1:] for file in files: if not os.path.isfile(file): @@ -26,5 +26,8 @@ uid = data['uid'] - r = cm.access({'action':'dockerfile', 'automation':'script', 'artifact': uid, 'quiet': 'yes'}) + r = cm.access({'action': 'dockerfile', + 'automation': 'script', + 'artifact': uid, + 'quiet': 'yes'}) checks.check_return(r) diff --git a/script/test-cm-core/src/script/process_readme.py b/script/test-cm-core/src/script/process_readme.py index e33f26528b..03daec7933 100644 --- a/script/test-cm-core/src/script/process_readme.py +++ b/script/test-cm-core/src/script/process_readme.py @@ -5,7 +5,7 @@ import json import yaml -files=sys.argv[1:] +files = sys.argv[1:] for file in files: if not os.path.isfile(file): @@ -22,5 +22,6 @@ data = yaml.safe_load(f) uid = data['uid'] - r = cm.access({'action':'doc', 'automation':'script', 'artifact': uid, 'quiet': 'yes'}) + r = cm.access({'action': 'doc', 'automation': 'script', + 'artifact': uid, 'quiet': 'yes'}) checks.check_return(r) diff --git a/script/test-cm-core/src/script/test_deps.py b/script/test-cm-core/src/script/test_deps.py index aaf19bc81b..37d75fd4c4 100644 --- a/script/test-cm-core/src/script/test_deps.py +++ b/script/test-cm-core/src/script/test_deps.py @@ -1,23 +1,25 @@ -# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps +# This test covers version, variation, compilation from src, add_deps, +# add_deps_recursive, deps, post_deps import cmind as cm import check as checks -# MLPerf v3.0 inference is now very outdated and we are testing inference in separate tests +# MLPerf v3.0 inference is now very outdated and we are testing inference +# in separate tests -#r = cm.access({'action':'run', 'automation':'script', 'tags': 'generate-run-cmds,mlperf', 'adr': +# r = cm.access({'action':'run', 'automation':'script', 'tags': 'generate-run-cmds,mlperf', 'adr': # {'loadgen': {'version': 'r3.0'}, 'compiler': {'tags': "gcc"}}, 'env': {'CM_MODEL': 'resnet50', # 'CM_DEVICE': 'cpu', 'CM_BACKEND': 'onnxruntime'}, 'quiet': 'yes'}) -#checks.check_return(r) +# checks.check_return(r) # -#r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'loadgen,version-r3.0,deps-python-non-virtual'}) -#checks.check_list(r, "loadgen,version-r3.0,deps-python-non-virtual") +# r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'loadgen,version-r3.0,deps-python-non-virtual'}) +# checks.check_list(r, "loadgen,version-r3.0,deps-python-non-virtual") # -#r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'inference,src,version-r3.0'}) -#checks.check_list(r, "inference,src,version-r3.0") +# r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'inference,src,version-r3.0'}) +# checks.check_list(r, "inference,src,version-r3.0") # -#r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'}) -#checks.check_return(r) +# r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'}) +# checks.check_return(r) # -#r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_tf,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'}) -#checks.check_return(r) +# r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_tf,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'}) +# checks.check_return(r) diff --git a/script/test-cm-core/src/script/test_features.py b/script/test-cm-core/src/script/test_features.py index d116cbd5bf..cde5e3ed21 100644 --- a/script/test-cm-core/src/script/test_features.py +++ b/script/test-cm-core/src/script/test_features.py @@ -5,17 +5,27 @@ import cmind as cm import check as checks -r = cm.access({'action':'run', 'automation':'script', 'tags': 'install,python-venv', 'name':'test', 'quiet': 'yes'}) +r = cm.access({'action': 'run', + 'automation': 'script', + 'tags': 'install,python-venv', + 'name': 'test', + 'quiet': 'yes'}) checks.check_return(r) -r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,python,virtual,name-test'}) +r = cm.access({'action': 'search', 'automation': 'cache', + 'tags': 'get,python,virtual,name-test'}) checks.check_list(r, "get,python-venv") -r = cm.access({'action':'run', 'automation':'script', 'tags': 'get,dataset,preprocessed,imagenet,_NHWC', 'quiet': 'yes'}) +r = cm.access({'action': 'run', + 'automation': 'script', + 'tags': 'get,dataset,preprocessed,imagenet,_NHWC', + 'quiet': 'yes'}) checks.check_return(r) -r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,dataset,preprocessed,imagenet,-_NCHW'}) +r = cm.access({'action': 'search', 'automation': 'cache', + 'tags': 'get,dataset,preprocessed,imagenet,-_NCHW'}) checks.check_list(r, "_NHWC") -r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,dataset,preprocessed,imagenet,-_NHWC'}) +r = cm.access({'action': 'search', 'automation': 'cache', + 'tags': 'get,dataset,preprocessed,imagenet,-_NHWC'}) checks.check_list(r, "_NHWC", False) diff --git a/script/test-cm-core/src/script/test_install.py b/script/test-cm-core/src/script/test_install.py index 66fa164d50..d4fb93ec70 100644 --- a/script/test-cm-core/src/script/test_install.py +++ b/script/test-cm-core/src/script/test_install.py @@ -3,8 +3,13 @@ import cmind as cm import check as checks -r = cm.access({'action':'run', 'automation':'script', 'tags': 'python,src,install,_shared', 'version': '3.9.10', 'quiet': 'true'}) +r = cm.access({'action': 'run', + 'automation': 'script', + 'tags': 'python,src,install,_shared', + 'version': '3.9.10', + 'quiet': 'true'}) checks.check_return(r) -r = cm.access({'action':'search', 'automation':'cache', 'tags': 'python,src,install,_shared,version-3.9.10'}) +r = cm.access({'action': 'search', 'automation': 'cache', + 'tags': 'python,src,install,_shared,version-3.9.10'}) checks.check_list(r, "python,src,install,_shared,version-3.9.10") diff --git a/script/test-cm-core/src/test_cm.py b/script/test-cm-core/src/test_cm.py index 41fb402c22..821e1571d6 100644 --- a/script/test-cm-core/src/test_cm.py +++ b/script/test-cm-core/src/test_cm.py @@ -3,12 +3,15 @@ r = cm.access(['test', 'script']) if 'return' not in r: - raise Exception('CM access function should always return key \'return\'!') + raise Exception( + 'CM access function should always return key \'return\'!') exit(0) except ImportError as e: from sys import stderr from subprocess import call - print('WARNING: CM module for python is not installed & jupyter notebooks will not be supported', file=stderr) + print( + 'WARNING: CM module for python is not installed & jupyter notebooks will not be supported', + file=stderr) retcode = call(['cm', 'test', 'script']) exit(retcode) diff --git a/script/test-cm-core/src/test_search_speed.py b/script/test-cm-core/src/test_search_speed.py index 3086a83408..577c4f0b80 100644 --- a/script/test-cm-core/src/test_search_speed.py +++ b/script/test-cm-core/src/test_search_speed.py @@ -5,21 +5,22 @@ steps = 10 -print ('Running search with tags {} times ...'.format(steps)) +print('Running search with tags {} times ...'.format(steps)) for step in range(steps): start = time.time() - r = cm.access({'action':'search', - 'automation':'script', - 'tags':'detect,os'}) + r = cm.access({'action': 'search', + 'automation': 'script', + 'tags': 'detect,os'}) timer = time.time() - start - if r['return']>0: cm.error(r) + if r['return'] > 0: + cm.error(r) times.append(timer) step = 0 for t in times: step += 1 - print ("{}) {:0.3f} sec.".format(step, t)) + print("{}) {:0.3f} sec.".format(step, t)) diff --git a/script/test-cm-core/src/tutorials/test_tutorial_retinanet.py b/script/test-cm-core/src/tutorials/test_tutorial_retinanet.py index 404b69fe93..eeb6f9e302 100644 --- a/script/test-cm-core/src/tutorials/test_tutorial_retinanet.py +++ b/script/test-cm-core/src/tutorials/test_tutorial_retinanet.py @@ -1,28 +1,37 @@ -# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps +# This test covers version, variation, compilation from src, add_deps, +# add_deps_recursive, deps, post_deps +import check as checks import cmind as cm from pathlib import Path import sys import os -sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script")) -import check as checks +sys.path.insert( + 1, + os.path.join( + Path(__file__).parent.parent.resolve(), + "script")) -r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu', 'adr': \ - {'python': {'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'scenario': 'Offline', \ - 'mode': 'accuracy', 'test_query_count': '10', 'rerun': 'true', 'quiet': 'yes'}) +r = cm.access({'action': 'run', 'automation': 'script', 'tags': 'app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu', 'adr': + {'python': {'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'scenario': 'Offline', + 'mode': 'accuracy', 'test_query_count': '10', 'rerun': 'true', 'quiet': 'yes'}) checks.check_return(r) -r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu', 'adr': \ - {'python': {'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'scenario': 'Offline', \ - 'mode': 'performance', 'test_query_count': '10', 'rerun': 'true', 'quiet': 'yes'}) +r = cm.access({'action': 'run', 'automation': 'script', 'tags': 'app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu', 'adr': + {'python': {'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'scenario': 'Offline', + 'mode': 'performance', 'test_query_count': '10', 'rerun': 'true', 'quiet': 'yes'}) checks.check_return(r) -r = cm.access({'action':'run', 'automation':'script', 'tags': 'install,python-venv', 'version': '3.10.8', 'name': 'mlperf' }) +r = cm.access({'action': 'run', + 'automation': 'script', + 'tags': 'install,python-venv', + 'version': '3.10.8', + 'name': 'mlperf'}) checks.check_return(r) -r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': \ - {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'submitter': 'Community', \ - 'implementation': 'cpp', 'hw_name': 'default', 'model': 'retinanet', 'backend': 'onnxruntime', 'device': 'cpu', 'scenario': 'Offline', \ - 'test_query_count': '10', 'clean': 'true', 'quiet': 'yes'}) +r = cm.access({'action': 'run', 'automation': 'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': + {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'submitter': 'Community', + 'implementation': 'cpp', 'hw_name': 'default', 'model': 'retinanet', 'backend': 'onnxruntime', 'device': 'cpu', 'scenario': 'Offline', + 'test_query_count': '10', 'clean': 'true', 'quiet': 'yes'}) checks.check_return(r) diff --git a/script/test-cm-script-pipeline/customize.py b/script/test-cm-script-pipeline/customize.py index 89311a22fa..d05326d931 100644 --- a/script/test-cm-script-pipeline/customize.py +++ b/script/test-cm-script-pipeline/customize.py @@ -3,13 +3,15 @@ from cmind import utils import os + def preprocess(i): - print ('') - print ('customize.py: preprocess') - print ('') + print('') + print('customize.py: preprocess') + print('') + + return {'return': 0} - return {'return':0} def postprocess(i): @@ -17,22 +19,22 @@ def postprocess(i): run_script_input = i['run_script_input'] env = i['env'] - print ('') - print ('customize.py: postprocess') - print ('') + print('') + print('customize.py: postprocess') + print('') - r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, 'script_name':'run2'}) - if r['return']>0: + r = automation.run_native_script( + {'run_script_input': run_script_input, 'env': env, 'script_name': 'run2'}) + if r['return'] > 0: return r - return {'return':0} + return {'return': 0} -def detect_version(i): +def detect_version(i): - print ('') - print ('customize.py: detect_version') - print ('') - + print('') + print('customize.py: detect_version') + print('') - return {'return':0} + return {'return': 0} diff --git a/script/test-cm-scripts/customize.py b/script/test-cm-scripts/customize.py index d12f9b3e1d..273999d460 100644 --- a/script/test-cm-scripts/customize.py +++ b/script/test-cm-scripts/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,10 +14,11 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/test-debug/customize.py b/script/test-debug/customize.py index a3af87f648..4c1388a1a3 100644 --- a/script/test-debug/customize.py +++ b/script/test-debug/customize.py @@ -2,29 +2,34 @@ import os + def preprocess(i): os_info = i['os_info'] env = i['env'] meta = i['meta'] - print ("********************************************************") - print ('- Importing CM library ...') + print("********************************************************") + print('- Importing CM library ...') import cmind - print (' SUCCESS!') - - cmind.utils.debug_here(__file__, port=5678, text='Debugging customize.py!', env=env, env_debug_uid='8d96cd9fa4734204').breakpoint() + print(' SUCCESS!') - print ('') - print ('- List CM repos ...') - print ('') - r = cmind.access({'action':'show', 'automation':'repo', 'out':'con'}) - print ('') - print (' SUCCESS!') - print ("********************************************************") + cmind.utils.debug_here( + __file__, + port=5678, + text='Debugging customize.py!', + env=env, + env_debug_uid='8d96cd9fa4734204').breakpoint() + print('') + print('- List CM repos ...') + print('') + r = cmind.access({'action': 'show', 'automation': 'repo', 'out': 'con'}) + print('') + print(' SUCCESS!') + print("********************************************************") - return {'return':0} + return {'return': 0} def postprocess(i): @@ -32,4 +37,4 @@ def postprocess(i): env = i['env'] state = i['state'] - return {'return':0} + return {'return': 0} diff --git a/script/test-download-and-extract-artifacts/customize.py b/script/test-download-and-extract-artifacts/customize.py index d12f9b3e1d..273999d460 100644 --- a/script/test-download-and-extract-artifacts/customize.py +++ b/script/test-download-and-extract-artifacts/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,10 +14,11 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/test-dummy/customize.py b/script/test-dummy/customize.py index d12f9b3e1d..273999d460 100644 --- a/script/test-dummy/customize.py +++ b/script/test-dummy/customize.py @@ -1,6 +1,7 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] @@ -13,10 +14,11 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') - return {'return':0} + return {'return': 0} + def postprocess(i): env = i['env'] - return {'return':0} + return {'return': 0} diff --git a/script/test-mlperf-inference-retinanet/customize.py b/script/test-mlperf-inference-retinanet/customize.py index 14e20d1bf2..1baf6756a2 100644 --- a/script/test-mlperf-inference-retinanet/customize.py +++ b/script/test-mlperf-inference-retinanet/customize.py @@ -1,13 +1,14 @@ from cmind import utils import os + def preprocess(i): os_info = i['os_info'] env = i['env'] meta = i['meta'] - return {'return':0} + return {'return': 0} def postprocess(i): @@ -15,4 +16,4 @@ def postprocess(i): env = i['env'] state = i['state'] - return {'return':0} + return {'return': 0} diff --git a/script/truncate-mlperf-inference-accuracy-log/customize.py b/script/truncate-mlperf-inference-accuracy-log/customize.py index d13d504ff8..fba057e493 100644 --- a/script/truncate-mlperf-inference-accuracy-log/customize.py +++ b/script/truncate-mlperf-inference-accuracy-log/customize.py @@ -4,6 +4,7 @@ import subprocess from os.path import exists + def preprocess(i): os_info = i['os_info'] @@ -12,14 +13,14 @@ def preprocess(i): if submission_dir == "": print("Please set CM_MLPERF_INFERENCE_SUBMISSION_DIR") - return {'return': 1, 'error':'CM_MLPERF_INFERENCE_SUBMISSION_DIR is not specified in env in run-mlperf-accuracy-log-truncator'} + return {'return': 1, 'error': 'CM_MLPERF_INFERENCE_SUBMISSION_DIR is not specified in env in run-mlperf-accuracy-log-truncator'} submitter = env.get("CM_MLPERF_SUBMITTER", "CTuning") os.system("rm -rf " + submission_dir + "_logs") CMD = env['CM_PYTHON_BIN'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "tools", "submission", - "truncate_accuracy_log.py") + "' --input '" + submission_dir + "' --submitter '" + submitter + "' --backup '" + submission_dir + "_logs'" + "truncate_accuracy_log.py") + "' --input '" + submission_dir + "' --submitter '" + submitter + "' --backup '" + submission_dir + "_logs'" env['CM_RUN_CMD'] = CMD - return {'return':0} + return {'return': 0} diff --git a/script/wrapper-reproduce-octoml-tinyml-submission/customize.py b/script/wrapper-reproduce-octoml-tinyml-submission/customize.py index f859dd1a9d..1e5beb50fb 100644 --- a/script/wrapper-reproduce-octoml-tinyml-submission/customize.py +++ b/script/wrapper-reproduce-octoml-tinyml-submission/customize.py @@ -2,6 +2,7 @@ import os import cmind as cm + def preprocess(i): os_info = i['os_info'] @@ -9,29 +10,34 @@ def preprocess(i): state = i['state'] inp = i['input'] if 'CM_FLASH_BOARD' in env: - script_tags = "flash,tiny" + script_tags = "flash,tiny" else: - script_tags = "reproduce,tiny,mlperf,octoml" - boards = ["NUCLEO", "NRF" ] - microtvm_variants = { "cmsis_nn": [ "ad", "ic", "vww", "kws" ], "native": [ "ic", "ad", "vww", "kws"] } + script_tags = "reproduce,tiny,mlperf,octoml" + boards = ["NUCLEO", "NRF"] + microtvm_variants = { + "cmsis_nn": [ + "ad", "ic", "vww", "kws"], "native": [ + "ic", "ad", "vww", "kws"]} for board in boards: for microtvm_variant in microtvm_variants: if board == "NRF" and microtvm_variant == "native": continue for model in microtvm_variants[microtvm_variant]: - variation_tags_string="_"+board+",_"+microtvm_variant+",_"+model + variation_tags_string = "_" + board + ",_" + microtvm_variant + ",_" + model tags = script_tags + "," + variation_tags_string if 'CM_RECREATE_BINARY' in env: - r = cm.access({'action':'rm', 'automation':'cache', 'tags': tags, 'force': 'true'}) + r = cm.access( + {'action': 'rm', 'automation': 'cache', 'tags': tags, 'force': 'true'}) if r['return'] > 0: return r - r = cm.access({'action':'run', 'automation':'script', 'tags': tags, 'quiet': 'true', 'env': env, - 'input': inp, 'state': state, 'add_deps': inp.get('add_deps', {}), 'add_deps_recursive': - inp.get('add_deps_recursive', {})}) + r = cm.access({'action': 'run', 'automation': 'script', 'tags': tags, 'quiet': 'true', 'env': env, + 'input': inp, 'state': state, 'add_deps': inp.get('add_deps', {}), 'add_deps_recursive': + inp.get('add_deps_recursive', {})}) if r['return'] > 0: return r - return {'return':0} + return {'return': 0} + def postprocess(i): - return {'return':0} + return {'return': 0} diff --git a/tests/script/process_dockerfile.py b/tests/script/process_dockerfile.py index 9a58fce389..d9abd2abeb 100644 --- a/tests/script/process_dockerfile.py +++ b/tests/script/process_dockerfile.py @@ -5,7 +5,7 @@ import json import yaml -files=sys.argv[1:] +files = sys.argv[1:] for file in files: if not os.path.isfile(file): @@ -26,5 +26,8 @@ uid = data['uid'] - r = cm.access({'action':'dockerfile', 'automation':'script', 'artifact': uid, 'quiet': 'yes'}) + r = cm.access({'action': 'dockerfile', + 'automation': 'script', + 'artifact': uid, + 'quiet': 'yes'}) checks.check_return(r) diff --git a/tests/script/process_readme.py b/tests/script/process_readme.py index 492813a050..de7e04033e 100644 --- a/tests/script/process_readme.py +++ b/tests/script/process_readme.py @@ -5,7 +5,7 @@ import json import yaml -files=sys.argv[1:] +files = sys.argv[1:] for file in files: if not os.path.isfile(file): @@ -22,5 +22,6 @@ data = yaml.safe_load(f) uid = data['uid'] - r = cm.access({'action':'doc', 'automation':'script', 'artifact': uid, 'quiet': 'yes'}) + r = cm.access({'action': 'doc', 'automation': 'script', + 'artifact': uid, 'quiet': 'yes'}) checks.check_return(r) diff --git a/tests/script/process_tests.py b/tests/script/process_tests.py index a9a7d0e555..8012d097b6 100644 --- a/tests/script/process_tests.py +++ b/tests/script/process_tests.py @@ -5,7 +5,7 @@ import json import yaml -files=sys.argv[1:] +files = sys.argv[1:] for file in files: print(file) @@ -20,12 +20,12 @@ elif file.endswith(".yaml"): data = yaml.safe_load(f) if data.get('uid', '') == '': - continue #not a CM script meta + continue # not a CM script meta uid = data['uid'] ii = { - 'action':'test', 'automation':'script', 'artifact': uid, 'quiet': 'yes', 'out': 'con' - } + 'action': 'test', 'automation': 'script', 'artifact': uid, 'quiet': 'yes', 'out': 'con' + } if os.environ.get('DOCKER_CM_REPO', '') != '': ii['docker_cm_repo'] = os.environ['DOCKER_CM_REPO'] if os.environ.get('DOCKER_CM_REPO_BRANCH', '') != '': diff --git a/tests/script/test_deps.py b/tests/script/test_deps.py index aaf19bc81b..37d75fd4c4 100644 --- a/tests/script/test_deps.py +++ b/tests/script/test_deps.py @@ -1,23 +1,25 @@ -# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps +# This test covers version, variation, compilation from src, add_deps, +# add_deps_recursive, deps, post_deps import cmind as cm import check as checks -# MLPerf v3.0 inference is now very outdated and we are testing inference in separate tests +# MLPerf v3.0 inference is now very outdated and we are testing inference +# in separate tests -#r = cm.access({'action':'run', 'automation':'script', 'tags': 'generate-run-cmds,mlperf', 'adr': +# r = cm.access({'action':'run', 'automation':'script', 'tags': 'generate-run-cmds,mlperf', 'adr': # {'loadgen': {'version': 'r3.0'}, 'compiler': {'tags': "gcc"}}, 'env': {'CM_MODEL': 'resnet50', # 'CM_DEVICE': 'cpu', 'CM_BACKEND': 'onnxruntime'}, 'quiet': 'yes'}) -#checks.check_return(r) +# checks.check_return(r) # -#r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'loadgen,version-r3.0,deps-python-non-virtual'}) -#checks.check_list(r, "loadgen,version-r3.0,deps-python-non-virtual") +# r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'loadgen,version-r3.0,deps-python-non-virtual'}) +# checks.check_list(r, "loadgen,version-r3.0,deps-python-non-virtual") # -#r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'inference,src,version-r3.0'}) -#checks.check_list(r, "inference,src,version-r3.0") +# r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'inference,src,version-r3.0'}) +# checks.check_list(r, "inference,src,version-r3.0") # -#r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'}) -#checks.check_return(r) +# r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'}) +# checks.check_return(r) # -#r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_tf,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'}) -#checks.check_return(r) +# r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_tf,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'}) +# checks.check_return(r) diff --git a/tests/script/test_features.py b/tests/script/test_features.py index 0679099321..b29ee6a7a5 100644 --- a/tests/script/test_features.py +++ b/tests/script/test_features.py @@ -5,23 +5,34 @@ import cmind as cm import check as checks -r = cm.access({'action':'run', 'automation':'script', 'tags': 'install,python-venv', 'name':'test', 'quiet': 'yes'}) +r = cm.access({'action': 'run', + 'automation': 'script', + 'tags': 'install,python-venv', + 'name': 'test', + 'quiet': 'yes'}) checks.check_return(r) -r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,python,virtual,name-test'}) +r = cm.access({'action': 'search', 'automation': 'cache', + 'tags': 'get,python,virtual,name-test'}) checks.check_list(r, "get,python-venv") -r = cm.access({'action':'run', 'automation':'script', 'tags': 'get,dataset,preprocessed,imagenet,_NHWC', 'quiet': 'yes'}) +r = cm.access({'action': 'run', + 'automation': 'script', + 'tags': 'get,dataset,preprocessed,imagenet,_NHWC', + 'quiet': 'yes'}) checks.check_return(r) -r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,dataset,preprocessed,imagenet,-_NCHW'}) +r = cm.access({'action': 'search', 'automation': 'cache', + 'tags': 'get,dataset,preprocessed,imagenet,-_NCHW'}) checks.check_list(r, "_NHWC") -r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,dataset,preprocessed,imagenet,-_NHWC'}) -#checks.check_list(r, "-_NHWC", False) +r = cm.access({'action': 'search', 'automation': 'cache', + 'tags': 'get,dataset,preprocessed,imagenet,-_NHWC'}) +# checks.check_list(r, "-_NHWC", False) -r = cm.access({'action':'run', 'automation': 'script', 'tags': 'test-scripts,_v1,_v2'}) +r = cm.access({'action': 'run', 'automation': 'script', + 'tags': 'test-scripts,_v1,_v2'}) new_env = r['new_env'] checks.check_key_value(new_env, "CM_VAR1", "combv1v2") checks.check_key_value(new_env, "CM_VAR2", "constv2") diff --git a/tests/script/test_install.py b/tests/script/test_install.py index 66fa164d50..d4fb93ec70 100644 --- a/tests/script/test_install.py +++ b/tests/script/test_install.py @@ -3,8 +3,13 @@ import cmind as cm import check as checks -r = cm.access({'action':'run', 'automation':'script', 'tags': 'python,src,install,_shared', 'version': '3.9.10', 'quiet': 'true'}) +r = cm.access({'action': 'run', + 'automation': 'script', + 'tags': 'python,src,install,_shared', + 'version': '3.9.10', + 'quiet': 'true'}) checks.check_return(r) -r = cm.access({'action':'search', 'automation':'cache', 'tags': 'python,src,install,_shared,version-3.9.10'}) +r = cm.access({'action': 'search', 'automation': 'cache', + 'tags': 'python,src,install,_shared,version-3.9.10'}) checks.check_list(r, "python,src,install,_shared,version-3.9.10") diff --git a/tests/test_cm.py b/tests/test_cm.py index 41fb402c22..821e1571d6 100644 --- a/tests/test_cm.py +++ b/tests/test_cm.py @@ -3,12 +3,15 @@ r = cm.access(['test', 'script']) if 'return' not in r: - raise Exception('CM access function should always return key \'return\'!') + raise Exception( + 'CM access function should always return key \'return\'!') exit(0) except ImportError as e: from sys import stderr from subprocess import call - print('WARNING: CM module for python is not installed & jupyter notebooks will not be supported', file=stderr) + print( + 'WARNING: CM module for python is not installed & jupyter notebooks will not be supported', + file=stderr) retcode = call(['cm', 'test', 'script']) exit(retcode) diff --git a/tests/test_search_speed.py b/tests/test_search_speed.py index 3086a83408..577c4f0b80 100644 --- a/tests/test_search_speed.py +++ b/tests/test_search_speed.py @@ -5,21 +5,22 @@ steps = 10 -print ('Running search with tags {} times ...'.format(steps)) +print('Running search with tags {} times ...'.format(steps)) for step in range(steps): start = time.time() - r = cm.access({'action':'search', - 'automation':'script', - 'tags':'detect,os'}) + r = cm.access({'action': 'search', + 'automation': 'script', + 'tags': 'detect,os'}) timer = time.time() - start - if r['return']>0: cm.error(r) + if r['return'] > 0: + cm.error(r) times.append(timer) step = 0 for t in times: step += 1 - print ("{}) {:0.3f} sec.".format(step, t)) + print("{}) {:0.3f} sec.".format(step, t)) diff --git a/tests/tutorials/test_tutorial_retinanet.py b/tests/tutorials/test_tutorial_retinanet.py index 9ecb2a3bef..dcca78f205 100644 --- a/tests/tutorials/test_tutorial_retinanet.py +++ b/tests/tutorials/test_tutorial_retinanet.py @@ -1,28 +1,37 @@ -# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps +# This test covers version, variation, compilation from src, add_deps, +# add_deps_recursive, deps, post_deps +import check as checks import cmind as cm from pathlib import Path import sys import os -sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script")) -import check as checks +sys.path.insert( + 1, + os.path.join( + Path(__file__).parent.parent.resolve(), + "script")) -r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu', 'adr': \ - {'python': {'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'scenario': 'Offline', \ - 'mode': 'accuracy', 'test_query_count': '10', 'rerun': 'true', 'quiet': 'yes'}) +r = cm.access({'action': 'run', 'automation': 'script', 'tags': 'app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu', 'adr': + {'python': {'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'scenario': 'Offline', + 'mode': 'accuracy', 'test_query_count': '10', 'rerun': 'true', 'quiet': 'yes'}) checks.check_return(r) -r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu', 'adr': \ - {'python': {'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'scenario': 'Offline', \ - 'mode': 'performance', 'test_query_count': '10', 'rerun': 'true', 'quiet': 'yes'}) +r = cm.access({'action': 'run', 'automation': 'script', 'tags': 'app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu', 'adr': + {'python': {'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'scenario': 'Offline', + 'mode': 'performance', 'test_query_count': '10', 'rerun': 'true', 'quiet': 'yes'}) checks.check_return(r) -r = cm.access({'action':'run', 'automation':'script', 'tags': 'install,python-venv', 'version': '3.10.8', 'name': 'mlperf' }) +r = cm.access({'action': 'run', + 'automation': 'script', + 'tags': 'install,python-venv', + 'version': '3.10.8', + 'name': 'mlperf'}) checks.check_return(r) -r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short', 'adr': \ - {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'submitter': 'Community', \ - 'implementation': 'cpp', 'hw_name': 'default', 'model': 'retinanet', 'backend': 'onnxruntime', 'device': 'cpu', 'scenario': 'Offline', \ - 'test_query_count': '10', 'clean': 'true', 'quiet': 'yes'}) +r = cm.access({'action': 'run', 'automation': 'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short', 'adr': + {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'submitter': 'Community', + 'implementation': 'cpp', 'hw_name': 'default', 'model': 'retinanet', 'backend': 'onnxruntime', 'device': 'cpu', 'scenario': 'Offline', + 'test_query_count': '10', 'clean': 'true', 'quiet': 'yes'}) checks.check_return(r)