From b3402f1dfb627791149c1f99634ffffe5207400f Mon Sep 17 00:00:00 2001 From: Matthew Durand Date: Mon, 28 Aug 2023 10:06:31 -0700 Subject: [PATCH] Add unit tests and refactor stages (#50) * Add unit tests and refactor stages * Fix aws cp cmd * Fixes - remove test stage - add regex for CODEBUILD_SRC_DIR - add pre_build to sonar stage * fix order of artifact download * update reports dir * fix regex * Fix coverage format --- .github/workflows/dss-sdk.yml | 16 +++++++++++++-- buildspec/dss-sdk-build.yml | 27 +++++++++++++++++++++++++ buildspec/dss-sdk-unit-test.yml | 15 ++++++++++++++ buildspec/dss-sdk.yml | 35 -------------------------------- buildspec/sonar-scanner.yml | 36 +++++++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 37 deletions(-) create mode 100644 buildspec/dss-sdk-build.yml create mode 100644 buildspec/dss-sdk-unit-test.yml delete mode 100644 buildspec/dss-sdk.yml create mode 100644 buildspec/sonar-scanner.yml diff --git a/.github/workflows/dss-sdk.yml b/.github/workflows/dss-sdk.yml index fa83b425..b8ac276f 100644 --- a/.github/workflows/dss-sdk.yml +++ b/.github/workflows/dss-sdk.yml @@ -15,10 +15,15 @@ on: - synchronize jobs: - dss-sdk: + build: + strategy: + matrix: + component: + - dss-sdk-build + - dss-sdk-unit-test uses: OpenMPDK/DSS/.github/workflows/build-aws.yml@master with: - component: dss-sdk + component: ${{ matrix.component }} project-name: OpenMPDK_dss-sdk compute-type-override: BUILD_GENERAL1_LARGE secrets: inherit @@ -33,3 +38,10 @@ jobs: component: ${{ matrix.component }} project-name: OpenMPDK_dss-sdk secrets: inherit + scan: + uses: OpenMPDK/DSS/.github/workflows/build-aws.yml@master + with: + component: sonar-scanner + project-name: OpenMPDK_dss-sdk + secrets: inherit + needs: [build] diff --git a/buildspec/dss-sdk-build.yml b/buildspec/dss-sdk-build.yml new file mode 100644 index 00000000..093f3e7e --- /dev/null +++ b/buildspec/dss-sdk-build.yml @@ -0,0 +1,27 @@ +version: 0.2 + +env: + secrets-manager: + DSSS3URI: Codebuild-DSS:DSSS3URI + variables: + DSSGLOBLIST: "nkv-sdk-*.tgz nkv-target-*.tgz" + +phases: + build: + commands: + # Build dss-sdk with Sonar build-wrapper for C/C++ static analysis + - build-wrapper-linux-x86-64 --out-dir bw-output ./scripts/build_all.sh kdd-samsung-remote + post_build: + commands: + # Copy artifacts to branch dir if this is a merge + - /stagemergeartifacts.sh + # Copy df_out, host_out, and bw-output for later sonar-scanner stage + - aws s3 cp --recursive bw-output "$DSSS3URI/cache/dss-sdk/$GITHUB_RUN_NUMBER/sonar/bw-output" --only-show-errors + - aws s3 cp --recursive df_out "$DSSS3URI/cache/dss-sdk/$GITHUB_RUN_NUMBER/build/df_out" --only-show-errors + - aws s3 cp --recursive host_out "$DSSS3URI/cache/dss-sdk/$GITHUB_RUN_NUMBER/build/host_out" --only-show-errors +artifacts: + files: + - df_out/nkv-target-*.tgz + - host_out/nkv-sdk-bin-*.tgz + discard-paths: yes + name: builds/dss-sdk/$GITHUB_RUN_NUMBER diff --git a/buildspec/dss-sdk-unit-test.yml b/buildspec/dss-sdk-unit-test.yml new file mode 100644 index 00000000..97948a78 --- /dev/null +++ b/buildspec/dss-sdk-unit-test.yml @@ -0,0 +1,15 @@ +version: 0.2 + +env: + secrets-manager: + DSSS3URI: Codebuild-DSS:DSSS3URI + +phases: + build: + commands: + # Build dss-sdk target with unit tests + - ./target/build.sh --run-tests --with-coverage + post_build: + commands: + # Copy df_out for later sonar-scanner stage + - aws s3 cp --recursive df_out "$DSSS3URI/cache/dss-sdk/$GITHUB_RUN_NUMBER/unit/df_out" --only-show-errors diff --git a/buildspec/dss-sdk.yml b/buildspec/dss-sdk.yml deleted file mode 100644 index 5100f93f..00000000 --- a/buildspec/dss-sdk.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: 0.2 - -env: - secrets-manager: - SONAR_TOKEN: Codebuild-DSS:SONAR_TOKEN - DSSS3URI: Codebuild-DSS:DSSS3URI - variables: - DSSGLOBLIST: "nkv-sdk-*.tgz nkv-target-*.tgz" - -phases: - build: - commands: - - | - build-wrapper-linux-x86-64 --out-dir bw-output \ - ./scripts/build_all.sh kdd-samsung-remote - - | - sonar-scanner \ - -Dsonar.branch.name="$([[ "$GITHUB_REF_NAME" != *"/merge" ]] && echo "$GITHUB_REF_NAME")" \ - -Dsonar.host.url=https://sonarcloud.io \ - -Dsonar.pullrequest.github.summary_comment=true \ - -Dsonar.pullrequest.provider=GitHub \ - -Dsonar.pullrequest.github.repository=OpenMPDK/dss-sdk \ - -Dsonar.pullrequest.key=$(echo $GITHUB_REF | grep -oP "^refs/pull/\K[^/]+") \ - -Dsonar.pullrequest.base=$GITHUB_BASE_REF \ - -Dsonar.pullrequest.branch=$GITHUB_HEAD_REF - post_build: - commands: - - /stagemergeartifacts.sh - -artifacts: - files: - - df_out/nkv-target-*.tgz - - host_out/nkv-sdk-bin-*.tgz - discard-paths: yes - name: builds/dss-sdk/$CODEBUILD_BUILD_NUMBER diff --git a/buildspec/sonar-scanner.yml b/buildspec/sonar-scanner.yml new file mode 100644 index 00000000..5d7b632b --- /dev/null +++ b/buildspec/sonar-scanner.yml @@ -0,0 +1,36 @@ +version: 0.2 + +env: + secrets-manager: + SONAR_TOKEN: Codebuild-DSS:SONAR_TOKEN + DSSS3URI: Codebuild-DSS:DSSS3URI + variables: + SONAR_UNIT_TEST_REPORT: df_out/reports/sonar_qube_ut_coverage_report.xml + +phases: + pre_build: + commands: + # Download build wrapper output from build job + - aws s3 cp --recursive "$DSSS3URI/cache/dss-sdk/$GITHUB_RUN_NUMBER/sonar" . --only-show-errors + # Download df_out and host_out + - aws s3 cp --recursive "$DSSS3URI/cache/dss-sdk/$GITHUB_RUN_NUMBER/build" . --only-show-errors + # Download unit test coverage report(s) + - aws s3 cp --recursive "$DSSS3URI/cache/dss-sdk/$GITHUB_RUN_NUMBER/unit/df_out/reports/" df_out/reports/ --only-show-errors + # Correct paths of target files in coverage report + - sed -i -r 's/path="/path="target\//g' $SONAR_UNIT_TEST_REPORT + # replace the old CODEBUILD_SRC_DIR with the current one in bw-output + - sed -i -r "s|/codebuild/output/src[^/]+/src/github.com/OpenMPDK/dss-sdk|$CODEBUILD_SRC_DIR|g" bw-output/build-wrapper-dump.json + build: + commands: + # Run sonar-scanner and ingest coverage report(s) + - | + sonar-scanner \ + -Dsonar.branch.name="$([[ "$GITHUB_REF_NAME" != *"/merge" ]] && echo "$GITHUB_REF_NAME")" \ + -Dsonar.host.url=https://sonarcloud.io \ + -Dsonar.pullrequest.github.summary_comment=true \ + -Dsonar.pullrequest.provider=GitHub \ + -Dsonar.pullrequest.github.repository=OpenMPDK/dss-sdk \ + -Dsonar.pullrequest.key=$(echo $GITHUB_REF | grep -oP "^refs/pull/\K[^/]+") \ + -Dsonar.pullrequest.base=$GITHUB_BASE_REF \ + -Dsonar.pullrequest.branch=$GITHUB_HEAD_REF \ + -Dsonar.coverageReportPaths=$SONAR_UNIT_TEST_REPORT