diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 54ee0fd7be..97969418c6 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -108,6 +108,8 @@ def preprocess(i): elif "no such file" in checksum_result.stderr.lower(): #print(f"No file {env['CM_DOWNLOAD_FILENAME']}. Downloading through cmutil.") cmutil_require_download = 1 + elif checksum_result.returncode == 1: + return {"return":1, "error":f"Error while checking checksum: {checksum_result.stderr}"} else: print(f"File {env['CM_DOWNLOAD_FILENAME']} already present, original checksum and computed checksum matches! Skipping Download..") else: diff --git a/script/download-file/run.sh b/script/download-file/run.sh index b738a7cd11..e0b9037c17 100644 --- a/script/download-file/run.sh +++ b/script/download-file/run.sh @@ -26,6 +26,8 @@ elif [ -e "${CM_DOWNLOAD_DOWNLOADED_PATH}" ]; then # checksum not supposed to fail for locally given file if [[ "${CM_DOWNLOAD_LOCAL_FILE_PATH}" != "" ]]; then exit 1 + else + CM_PRE_DOWNLOAD_CLEAN=true fi else require_download="0" @@ -35,7 +37,7 @@ fi if [[ ${require_download} == "1" ]]; then echo "" - if [ -e "${CM_PRE_DOWNLOAD_CLEAN}" ]; then + if [ "${CM_PRE_DOWNLOAD_CLEAN}" != "" ] && [ "${CM_PRE_DOWNLOAD_CLEAN,,}" != "false" ]; then echo ${CM_PRE_DOWNLOAD_CLEAN_CMD} ${CM_PRE_DOWNLOAD_CLEAN_CMD} fi diff --git a/script/get-ml-model-dlrm-terabyte/_cm.json b/script/get-ml-model-dlrm-terabyte/_cm.json index 553808932d..e9cb28c560 100644 --- a/script/get-ml-model-dlrm-terabyte/_cm.json +++ b/script/get-ml-model-dlrm-terabyte/_cm.json @@ -68,7 +68,8 @@ "CM_PACKAGE_URL": "https://dlrm.s3-us-west-1.amazonaws.com/models/tb00_40M.onnx.tar", "CM_UNTAR": "yes", "CM_ML_MODEL_FILE": "tb00_40M.onnx", - "CM_ML_MODEL_DLRM_MAX_INDEX_RANGE": "40000000" + "CM_ML_MODEL_DLRM_MAX_INDEX_RANGE": "40000000", + "CM_DOWNLOAD_CHECKSUM": "763b964eaffe5f86e92cdcb60c5dc0de" } }, "pytorch": { @@ -138,7 +139,8 @@ "env": { "CM_ML_MODEL_ACCURACY": "0.8107", "CM_PACKAGE_URL": "https://dlrm.s3-us-west-1.amazonaws.com/models/tb0875_10M.pt", - "CM_ML_MODEL_DLRM_MAX_INDEX_RANGE": "10000000" + "CM_ML_MODEL_DLRM_MAX_INDEX_RANGE": "10000000", + "CM_DOWNLOAD_CHECKSUM": "b7cacffcf75f767faa9cb2af397723aa" } }, "onnx,fp32,debug": { @@ -147,7 +149,8 @@ "CM_PACKAGE_URL": "https://dlrm.s3-us-west-1.amazonaws.com/models/tb0875_10M.onnx.tar", "CM_ML_MODEL_DLRM_MAX_INDEX_RANGE": "10000000", "CM_UNTAR": "yes", - "CM_ML_MODEL_FILE": "tb0875_10M.onnx" + "CM_ML_MODEL_FILE": "tb0875_10M.onnx", + "CM_DOWNLOAD_CHECKSUM": "d11255cd9926cda9181a347861e4d263" } }, "weight_sharded": {