Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DLRM model checksum + corrected pre download clean #125

Merged
merged 14 commits into from
Sep 24, 2024
Merged

DLRM model checksum + corrected pre download clean #125

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
de42748
added checksums
anandhu-eng Sep 21, 2024
7249b73
corrected pre download clean
anandhu-eng Sep 21, 2024
b2b7dfc
Disabled check for condition
anandhu-eng Sep 21, 2024
a6224e3
Merge branch 'mlperf-inference' into checksum-branch-1
anandhu-eng Sep 21, 2024
67dc948
Proper exit for unhandled md5sum errors
anandhu-eng Sep 23, 2024
44076fb
Merge branch 'mlperf-inference' into checksum-branch-1
anandhu-eng Sep 23, 2024
af58441
Merge branch 'mlperf-inference' into checksum-branch-1
anandhu-eng Sep 24, 2024
d12083e
preclean fixed
anandhu-eng Sep 24, 2024
74030b2
deleted checksum for url -> cloud.*
anandhu-eng Sep 24, 2024
3566ac1
proper handling of pre_clean
anandhu-eng Sep 24, 2024
27861a5
reverted pre clean change
anandhu-eng Sep 24, 2024
0c98cbd
code clean
anandhu-eng Sep 24, 2024
b40ea46
fix pre download clean
anandhu-eng Sep 24, 2024
29bed25
handled false condition in download-file
anandhu-eng Sep 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions script/download-file/customize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,8 @@ def preprocess(i):
elif "no such file" in checksum_result.stderr.lower():
#print(f"No file {env['CM_DOWNLOAD_FILENAME']}. Downloading through cmutil.")
cmutil_require_download = 1
elif checksum_result.returncode == 1:
return {"return":1, "error":f"Error while checking checksum: {checksum_result.stderr}"}
else:
print(f"File {env['CM_DOWNLOAD_FILENAME']} already present, original checksum and computed checksum matches! Skipping Download..")
else:
Expand Down
4 changes: 3 additions & 1 deletion script/download-file/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ elif [ -e "${CM_DOWNLOAD_DOWNLOADED_PATH}" ]; then
# checksum not supposed to fail for locally given file
if [[ "${CM_DOWNLOAD_LOCAL_FILE_PATH}" != "" ]]; then
exit 1
else
CM_PRE_DOWNLOAD_CLEAN=true
fi
else
require_download="0"
Expand All @@ -35,7 +37,7 @@ fi

if [[ ${require_download} == "1" ]]; then
echo ""
if [ -e "${CM_PRE_DOWNLOAD_CLEAN}" ]; then
if [ "${CM_PRE_DOWNLOAD_CLEAN}" != "" ]; then
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should check for not equal to "False" here

echo ${CM_PRE_DOWNLOAD_CLEAN_CMD}
${CM_PRE_DOWNLOAD_CLEAN_CMD}
fi
Expand Down
9 changes: 6 additions & 3 deletions script/get-ml-model-dlrm-terabyte/_cm.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@
"CM_PACKAGE_URL": "https://dlrm.s3-us-west-1.amazonaws.com/models/tb00_40M.onnx.tar",
"CM_UNTAR": "yes",
"CM_ML_MODEL_FILE": "tb00_40M.onnx",
"CM_ML_MODEL_DLRM_MAX_INDEX_RANGE": "40000000"
"CM_ML_MODEL_DLRM_MAX_INDEX_RANGE": "40000000",
"CM_DOWNLOAD_CHECKSUM": "763b964eaffe5f86e92cdcb60c5dc0de"
}
},
"pytorch": {
Expand Down Expand Up @@ -138,7 +139,8 @@
"env": {
"CM_ML_MODEL_ACCURACY": "0.8107",
"CM_PACKAGE_URL": "https://dlrm.s3-us-west-1.amazonaws.com/models/tb0875_10M.pt",
"CM_ML_MODEL_DLRM_MAX_INDEX_RANGE": "10000000"
"CM_ML_MODEL_DLRM_MAX_INDEX_RANGE": "10000000",
"CM_DOWNLOAD_CHECKSUM": "b7cacffcf75f767faa9cb2af397723aa"
}
},
"onnx,fp32,debug": {
Expand All @@ -147,7 +149,8 @@
"CM_PACKAGE_URL": "https://dlrm.s3-us-west-1.amazonaws.com/models/tb0875_10M.onnx.tar",
"CM_ML_MODEL_DLRM_MAX_INDEX_RANGE": "10000000",
"CM_UNTAR": "yes",
"CM_ML_MODEL_FILE": "tb0875_10M.onnx"
"CM_ML_MODEL_FILE": "tb0875_10M.onnx",
"CM_DOWNLOAD_CHECKSUM": "d11255cd9926cda9181a347861e4d263"
}
},
"weight_sharded": {
Expand Down
Loading