Skip to content

Commit

Permalink
DEVPROD-12110 Use assumed credentials for s3 (#534)
Browse files Browse the repository at this point in the history
  • Loading branch information
blink1073 authored Oct 28, 2024
1 parent fecfe59 commit b4714b0
Showing 1 changed file with 84 additions and 50 deletions.
134 changes: 84 additions & 50 deletions .evergreen/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,17 @@ functions:
file: src/expansion.yml

"upload release":
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: ${project}.tar.gz
remote_file: ${UPLOAD_BUCKET}/${project}-${CURRENT_VERSION}.tar.gz
bucket: mciuploads
remote_file: ${project}-${CURRENT_VERSION}.tar.gz
bucket: ${aws_bucket}
permissions: public-read
content_type: ${content_type|application/x-gzip}

Expand All @@ -76,14 +80,18 @@ functions:
source_dir: ${PROJECT_DIRECTORY}/
include:
- "./**"
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: ${build_id}.tar.gz
# Example: /mciuploads/${UPLOAD_BUCKET}/gcc49/9dfb7d741efbca16faa7859b9349d7a942273e43/debug-compile-nosasl-nossl/mongo_c_driver_releng_9dfb7d741efbca16faa7859b9349d7a942273e43_16_11_08_19_29_52.tar.gz
remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${branch_name}/${task_name}/build.tar.gz
bucket: mciuploads
# Example: /${aws_bucket}/gcc49/9dfb7d741efbca16faa7859b9349d7a942273e43/debug-compile-nosasl-nossl/mongo_c_driver_releng_9dfb7d741efbca16faa7859b9349d7a942273e43_16_11_08_19_29_52.tar.gz
remote_file: ${build_variant}/${revision}/${branch_name}/${task_name}/build.tar.gz
bucket: ${aws_bucket}
permissions: public-read
content_type: ${content_type|application/x-gzip}

Expand All @@ -92,12 +100,16 @@ functions:
params:
continue_on_err: true
script: "rm -rf ${PROJECT_DIRECTORY}"
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
- command: s3.get
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${branch_name}/${BUILD_NAME}/build.tar.gz
bucket: mciuploads
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
remote_file: ${build_variant}/${revision}/${branch_name}/${BUILD_NAME}/build.tar.gz
bucket: ${aws_bucket}
extract_to: ${PROJECT_DIRECTORY}

"exec compile script" :
Expand All @@ -117,39 +129,45 @@ functions:
[ -f ${PROJECT_DIRECTORY}/${file} ] && bash ${PROJECT_DIRECTORY}/${file} || echo "${PROJECT_DIRECTORY}/${file} not available, skipping"
"upload docs" :
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
- command: shell.exec
params:
silent: true
include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
script: |
export AWS_ACCESS_KEY_ID=${aws_key}
export AWS_SECRET_ACCESS_KEY=${aws_secret}
aws s3 cp ${PROJECT_DIRECTORY}/doc/html s3://mciuploads/${UPLOAD_BUCKET}/docs/${CURRENT_VERSION} --recursive --acl public-read --region us-east-1
aws s3 cp ${PROJECT_DIRECTORY}/doc/html s3://${aws_bucket}/docs/${CURRENT_VERSION} --recursive --acl public-read --region us-east-1
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: ${PROJECT_DIRECTORY}/doc/html/index.html
remote_file: ${UPLOAD_BUCKET}/docs/${CURRENT_VERSION}/index.html
bucket: mciuploads
remote_file: docs/${CURRENT_VERSION}/index.html
bucket: ${aws_bucket}
permissions: public-read
content_type: text/html
display_name: "Rendered docs"

"upload coverage" :
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
- command: shell.exec
params:
silent: true
include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
script: |
export AWS_ACCESS_KEY_ID=${aws_key}
export AWS_SECRET_ACCESS_KEY=${aws_secret}
aws s3 cp ${PROJECT_DIRECTORY}/coverage s3://mciuploads/${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/coverage/ --recursive --acl public-read --region us-east-1
aws s3 cp ${PROJECT_DIRECTORY}/coverage s3://${aws_bucket}/${build_variant}/${revision}/${version_id}/${build_id}/coverage/ --recursive --acl public-read --region us-east-1
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: ${PROJECT_DIRECTORY}/coverage/index.html
remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/coverage/index.html
bucket: mciuploads
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/coverage/index.html
bucket: ${aws_bucket}
permissions: public-read
content_type: text/html
display_name: "Coverage Report"
Expand All @@ -165,43 +183,51 @@ functions:
else
echo "No issues found" > scan.html
fi
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
- command: shell.exec
params:
silent: true
include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
script: |
export AWS_ACCESS_KEY_ID=${aws_key}
export AWS_SECRET_ACCESS_KEY=${aws_secret}
aws s3 cp ${PROJECT_DIRECTORY}/scan s3://mciuploads/${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/scan/ --recursive --acl public-read --region us-east-1
aws s3 cp ${PROJECT_DIRECTORY}/scan s3://${aws_bucket}/${build_variant}/${revision}/${version_id}/${build_id}/scan/ --recursive --acl public-read --region us-east-1
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: ${PROJECT_DIRECTORY}/scan.html
remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/scan/index.html
bucket: mciuploads
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/scan/index.html
bucket: ${aws_bucket}
permissions: public-read
content_type: text/html
display_name: "Scan Build Report"

"upload logs":
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: ${DRIVERS_TOOLS}/.evergreen/test_logs.tar.gz
remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-logs.tar.gz
bucket: mciuploads
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-logs.tar.gz
bucket: ${aws_bucket}
permissions: public-read
content_type: ${content_type|application/x-gzip}
display_name: "test_logs.tar.gz"
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
optional: true
local_file: ${DRIVERS_TOOLS}/.evergreen/orchestration/server.log
remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-orchestration.log
bucket: mciuploads
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-orchestration.log
bucket: ${aws_bucket}
permissions: public-read
content_type: ${content_type|text/plain}
display_name: "orchestration.log"
Expand All @@ -213,13 +239,17 @@ functions:
source_dir: ${PROJECT_DIRECTORY}/
include:
- "./**"
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: working-dir.tar.gz
remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-working-dir.tar.gz
bucket: mciuploads
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-working-dir.tar.gz
bucket: ${aws_bucket}
permissions: public-read
content_type: ${content_type|application/x-gzip}
display_name: "working-dir.tar.gz"
Expand All @@ -232,13 +262,17 @@ functions:
exclude_files:
# Windows cannot read the mongod *.lock files because they are locked.
- "*.lock"
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: drivers-dir.tar.gz
remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-drivers-dir.tar.gz
bucket: mciuploads
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-drivers-dir.tar.gz
bucket: ${aws_bucket}
permissions: public-read
content_type: ${content_type|application/x-gzip}
display_name: "drivers-dir.tar.gz"
Expand Down

0 comments on commit b4714b0

Please sign in to comment.