Merge pull request mlcommons#356 from GATEOverflow/mlperf-inference

Fix docker image naming SCC24, extended CM script tests

script/run-mlperf-inference-app/customize.py

@@ -23,6 +23,12 @@ def preprocess(i):
     if env.get('CM_RUN_DOCKER_CONTAINER', '') == "yes": 
         return {'return':0}
 
+    if env.get('CM_DOCKER_IMAGE_NAME', '') ==  'scc24':
+        if env.get("CM_MLPERF_IMPLEMENTATION", "reference") == "reference":
+            env['CM_DOCKER_IMAGE_NAME'] = "scc24-reference"
+        elif "nvidia" in env.get("CM_MLPERF_IMPLEMENTATION", "reference"):
+            env['CM_DOCKER_IMAGE_NAME'] = "scc24-nvidia"
+
     dump_version_info = env.get('CM_DUMP_VERSION_INFO', True)
 
     system_meta = state.get('CM_SUT_META', {})

script/test-cm-scripts/_cm.yaml

@@ -0,0 +1,31 @@
+alias: test-cm-scripts
+automation_alias: script
+automation_uid: 5b4e0237da074764
+cache: false
+tags:
+- test
+- test-scripts
+- cm-test
+env:
+  CM_VAR1: orig
+const:
+  CM_CVAR1: orig
+new_env_keys:
+  - CM_VAR*
+  - CM_CVAR*
+
+uid: 6fbe3884575c4e51
+variations:
+  v1:
+    env:
+      CM_VAR1: v1
+  v2:
+    env:
+      CM_VAR1: v2
+      CM_VAR2: v2
+    const:
+      CM_VAR2: constv2
+  v1,v2:
+    env:
+      CM_VAR1: combv1v2
+      CM_VAR2: combv1v2

script/test-cm-scripts/customize.py

@@ -0,0 +1,22 @@
+from cmind import utils
+import os
+
+def preprocess(i):
+
+    os_info = i['os_info']
+
+    env = i['env']
+
+    meta = i['meta']
+
+    automation = i['automation']
+
+    quiet = (env.get('CM_QUIET', False) == 'yes')
+
+    return {'return':0}
+
+def postprocess(i):
+
+    env = i['env']
+
+    return {'return':0}

script/test-cm-scripts/run.bat

@@ -0,0 +1 @@
+rem native script

script/test-cm-scripts/run.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+#CM Script location: ${CM_TMP_CURRENT_SCRIPT_PATH}
+
+#To export any variable
+#echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out
+
+#${CM_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency
+
+echo "Running: "
+echo "${CM_RUN_CMD}"
+echo ""
+
+if [[ ${CM_FAKE_RUN} != "yes" ]]; then
+  eval "${CM_RUN_CMD}"
+  test $? -eq 0 || exit 1
+fi

tests/script/check.py

@@ -12,3 +12,13 @@ def check_list(r, string, found=True):
         raise Exception('CM search returned an empty list for ' + string)
     if len(r['list']) > 0 and not found:
         raise Exception('CM search returned at lease one entry for ' + string)
+
+def check_key_value(d, key, value, absent_ok=False):
+    if not d.get(key):
+        if absent_ok:
+            return True
+        else:
+            raise Exception(f"{key} is missing. Current values are {d}")
+    elif d[key] != value:
+        raise Exception(f"{key} is not having the expected value of {value}. Current value is {d[key]}")
+

tests/script/test_features.py

@@ -18,4 +18,10 @@
 checks.check_list(r, "_NHWC")
 
 r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,dataset,preprocessed,imagenet,-_NHWC'})
-checks.check_list(r, "_NHWC", False)
+#checks.check_list(r, "-_NHWC", False)
+
+
+r = cm.access({'action':'run', 'automation': 'script', 'tags': 'test-scripts,_v1,_v2'})
+new_env = r['new_env']
+checks.check_key_value(new_env, "CM_VAR1", "combv1v2")
+checks.check_key_value(new_env, "CM_VAR2", "constv2")