Merged master into this branch.

mojaloop · May 11, 2020 · c9cde80 · c9cde80
1 parent 76e654a
commit c9cde80
Show file tree

Hide file tree

Showing 12 changed files with 4,166 additions and 4,561 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -7,8 +7,9 @@ version: 2.1
 # Orbs used in this pipeline
 ###
 orbs:
-  anchore: anchore/[email protected].0
+  anchore: anchore/[email protected].6
   deploy-kube: mojaloop/[email protected]
+  slack: circleci/[email protected]
 
 ##
 # defaults
@@ -50,7 +51,7 @@ executors:
   default-docker:
     working_directory: /home/circleci/project
     docker: 
-      - image: node:12.16.0-alpine
+      - image: node:12.16.1-alpine
 
   default-machine:
     machine:
@@ -136,7 +137,6 @@ jobs:
             else
                 echo "Not a release (env CIRCLE_BRANCH != 'master'), skipping sending lcov.info to SonarQube."
             fi
-
   vulnerability-check:
     executor: default-docker
     steps:
@@ -206,7 +206,7 @@ jobs:
           <<: *defaults_license_scanner
       - run:
           name: Run the license-scanner
-          command: cd /tmp/license-scanner && mode=docker dockerImage=$DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG make run
+          command: cd /tmp/license-scanner && mode=docker dockerImages=$DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG make run
       - store_artifacts:
           path: /tmp/license-scanner/results
           prefix: licenses
@@ -216,6 +216,13 @@ jobs:
     steps:
       - setup_remote_docker
       - checkout
+      - run:
+          name: Install docker dependencies for anchore
+          command: |
+            apk add --update py-pip docker python-dev libffi-dev openssl-dev gcc libc-dev make jq npm
+      - run:
+          name: Install general dependencies
+          command: *defaults_Dependencies
       - run:
           name: Install AWS CLI dependencies
           command: *defaults_awsCliDependencies
@@ -224,41 +231,39 @@ jobs:
       - run:
           name: Load the pre-built docker image from workspace
           command: docker load -i /tmp/docker-image.tar
+      - run:
+          name: Download the mojaloop/ci-config repo
+          command: |
+            git clone https://github.com/mojaloop/ci-config /tmp/ci-config
+            # Generate the mojaloop anchore-policy
+            cd /tmp/ci-config/container-scanning && ./mojaloop-policy-generator.js /tmp/mojaloop-policy.json
+      - run:
+          name: Pull base image locally
+          command: |
+            docker pull node:12.16.1-alpine
+      # Analyze the base and derived image
+      # Note: It seems images are scanned in parallel, so preloading the base image result doesn't give us any real performance gain
       - anchore/analyze_local_image:
-          dockerfile_path: ./Dockerfile
-          image_name: ${DOCKER_ORG}/${CIRCLE_PROJECT_REPONAME}:${CIRCLE_TAG}
-          # Anchore bug: if policy_failure is `true`, reports don't get written - we manually check for failures below
+          # Force the older version, version 0.7.0 was just published, and is broken
+          anchore_version: v0.6.1
+          image_name: "docker.io/node:12.16.1-alpine $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG"
           policy_failure: false
           timeout: '500'
-      - run:
-          name: Evaluate Failures.
-          command: |
-            if [[ ! $(which jq) ]]; then
-              (set +o pipefail; apk add jq || apt-get install -y jq || yum install -y jq)
-            fi
-            if [[ $(ls anchore-reports/*content-os*.json 2> /dev/null) ]]; then
-              printf "\n%s\n" "The following OS packages are installed:"
-              jq '[.content | sort_by(.package) | .[] | {package: .package, version: .version}]' anchore-reports/*content-os*.json
-            fi
-            if [[ $(ls anchore-reports/*vuln*.json 2> /dev/null) ]]; then
-              printf "\n%s\n" "The following vulnerabilities were found:"
-              jq '[.vulnerabilities | group_by(.package) | .[] | {package: .[0].package, vuln: [.[].vuln]}]' anchore-reports/*vuln*.json
-            fi
+          # Note: if the generated policy is invalid, this will fallback to the default policy, which we don't want!
+          policy_bundle_file_path: /tmp/mojaloop-policy.json
       - run:
           name: Upload Anchore reports to s3
           command: |
             aws s3 cp anchore-reports ${AWS_S3_DIR_ANCHORE_REPORTS}/${CIRCLE_PROJECT_REPONAME}/ --recursive
             aws s3 rm ${AWS_S3_DIR_ANCHORE_REPORTS}/latest/ --recursive --exclude "*" --include "${CIRCLE_PROJECT_REPONAME}*"
             aws s3 cp anchore-reports ${AWS_S3_DIR_ANCHORE_REPORTS}/latest/ --recursive
-        
-            # TODO: Enable this when we want to increase the strictness of our security policies
-            # failCount=$(cat anchore-reports/*policy*.json | grep 'fail' | wc -l)
-            # echo "FailCount is: ${failCount}"
-            # if [ $failCount -gt 0 ]; then
-            #   printf "Failed with a policy failure count of: ${failCount}"
-            #   exit 1
-            # fi
-
+      - run:
+          name: Evaluate failures
+          command: /tmp/ci-config/container-scanning/anchore-result-diff.js anchore-reports/node_12.16.1-alpine-policy.json anchore-reports/${CIRCLE_PROJECT_REPONAME}*-policy.json
+      - slack/status:
+          fail_only: true
+          webhook: "$SLACK_WEBHOOK_ANNOUNCEMENT"
+          failure_message: 'Anchore Image Scan failed  for: \`"${DOCKER_ORG}/${CIRCLE_PROJECT_REPONAME}:${CIRCLE_TAG}"\`'
       - store_artifacts:
           path: anchore-reports
 
@@ -285,16 +290,10 @@ jobs:
             docker push $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG
             echo "Publishing $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$RELEASE_TAG"
             docker push $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:$RELEASE_TAG
-      - run:
-          name: Slack announcement for tag releases
-          command: |
-            curl -X POST \
-              $SLACK_WEBHOOK_ANNOUNCEMENT \
-              -H 'Content-type: application/json' \
-              -H 'cache-control: no-cache' \
-              -d "{\"text\": \"*${CIRCLE_PROJECT_REPONAME}* - Release \`${CIRCLE_TAG}\`: https://github.com/mojaloop/${CIRCLE_PROJECT_REPONAME}/releases/tag/${CIRCLE_TAG}\"}"
-
-        
+      - slack/status:
+          webhook: "$SLACK_WEBHOOK_ANNOUNCEMENT"
+          success_message: '*"${CIRCLE_PROJECT_REPONAME}"* - Release \`"${CIRCLE_TAG}"\` \nhttps://github.com/mojaloop/"${CIRCLE_PROJECT_REPONAME}"/releases/tag/"${CIRCLE_TAG}"'
+
   deploy:
     executor: deploy-kube/helm-kube
     steps:
@@ -303,7 +302,10 @@ jobs:
           helm_set_values: |
             --set quoting-service.image.repository=$DOCKER_ORG/$CIRCLE_PROJECT_REPONAME \
             --set quoting-service.image.tag=$CIRCLE_TAG
-
+      - slack/status:
+          fail_only: true
+          webhook: "$SLACK_WEBHOOK_ANNOUNCEMENT"
+          failure_message: 'TEST Deployment failed for: \`"${DOCKER_ORG}/${CIRCLE_PROJECT_REPONAME}:${CIRCLE_TAG}"\`'
 ##
 # Workflows
 #
@@ -369,7 +371,6 @@ workflows:
       - build:
           context: org-global
           requires:
-            - setup
             - test-unit
             - test-coverage
             - vulnerability-check
@@ -405,7 +406,6 @@ workflows:
           requires:
             - license-scan
             - image-scan
-            - build
           filters:
             tags:
               only: /v[0-9]+(\.[0-9]+)*(\-snapshot)?(\-hotfix(\.[0-9]+))?/
@@ -421,4 +421,4 @@ workflows:
               only: /v[0-9]+(\.[0-9]+)*(\-snapshot)?/
             branches:
               ignore:
-                - /.*/
+                - /.*/
diff --git a/.ncurc.json b/.ncurc.json
@@ -1,6 +1,7 @@
 {
   "reject": [
     "@hapi/hapi",
-    "json-rules-engine"
+    "json-rules-engine",
+    "hapi-openapi"
   ]
 }
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:12.16.0-alpine as builder
+FROM node:12.16.1-alpine as builder
 
 WORKDIR /opt/quoting-service
 
@@ -15,11 +15,20 @@ RUN apk del build-dependencies
 
 COPY src /opt/quoting-service/src
 
-FROM node:12.16.0-alpine
+FROM node:12.16.1-alpine
 
 WORKDIR /opt/quoting-service
 
-COPY --from=builder /opt/quoting-service .
+# Create empty log file & link stdout to the application log file
+RUN mkdir ./logs && touch ./logs/combined.log
+# Links combined to stdout
+RUN ln -sf /dev/stdout ./logs/combined.log
+
+# Create a non-root user: ml-user
+RUN adduser -D ml-user 
+USER ml-user
+
+COPY --chown=ml-user --from=builder /opt/quoting-service .
 RUN npm prune --production
 
 EXPOSE 3002

diff --git a/README.md b/README.md
@@ -2,32 +2,35 @@
 [![Git Commit](https://img.shields.io/github/last-commit/mojaloop/quoting-service.svg?style=flat)](https://github.com/mojaloop/quoting-service/commits/master)
 [![Git Releases](https://img.shields.io/github/release/mojaloop/quoting-service.svg?style=flat)](https://github.com/mojaloop/quoting-service/releases)
 [![Docker pulls](https://img.shields.io/docker/pulls/mojaloop/quoting-service.svg?style=flat)](https://hub.docker.com/r/mojaloop/quoting-service)
-[![CircleCI](https://circleci.com/gh/mojaloop/quoting-service.svg?style=svg)](https://circleci.com/gh/mojaloop/quoting-service)
+[![CircleCI](https://circleci.com/gh/mojaloop/quoting-service.svg?style=svg)](https://app.circleci.com/pipelines/github/mojaloop/quoting-service)
+
+The Quoting Service was donated by the Mowali project working in collaboration with Orange and MTN. The original author of this service is James Bush ([email protected]).
 
-The Quoting Service was donated by the Mowali project working in collaboration with Orange and MTN. 
 The Quoting service is now part of the Mojaloop project and deployment.
 
 The service provided by the API resource /quotes is calculation of possible fees and FSP commission involved in performing an interoperable financial transaction. 
 Both the Payer and Payee FSP should calculate their part of the quote to be able to get a total view of all the fees and FSP commission involved in the transaction.
 
-### Contents:
+## Contents:
 
 - [Services Sequence overview](#services-sequence-overview)
-- [Local Deployment](#local-deployment)
+- [Running Locally](#running-locally)
+- [Auditing Dependencies](#auditing-dependencies)
+- [Container Scans](#container-scans)
 
 ## Services Sequence overview
 
 ![Quoting Service Sequence diagram](diagrams/quotingServiceSequences.svg)
 
 * [Quoting Service Sequence diagram](diagrams/quotingServiceSequences.puml)
 
-## Local Deployment
+## Running Locally
 
 Please follow the instruction in [Onboarding Document](onboarding.md) to setup and run the service locally.
 
 ## Auditing Dependencies
 
-We use `npm-audit-resolver` along with `npm audit` to check dependencies for vulnerabilities, and keep track of resolved dependencies with an `audit-resolv.json` file.
+We use `npm-audit-resolver` along with `npm audit` to check dependencies for node vulnerabilities, and keep track of resolved dependencies with an `audit-resolve.json` file.
 
 To start a new resolution process, run:
 ```bash
@@ -39,5 +42,15 @@ You can then check to see if the CI will pass based on the current dependencies
 npm run audit:check
 ```
 
-And commit the changed `audit-resolv.json` to ensure that CircleCI will build correctly.
-test
+And commit the changed `audit-resolve.json` to ensure that CircleCI will build correctly.
+
+## Container Scans
+
+As part of our CI/CD process, we use anchore-cli to scan our built docker container for vulnerabilities upon release.
+
+If you find your release builds are failing, refer to the [container scanning](https://github.com/mojaloop/ci-config#container-scanning) in our shared Mojaloop CI config repo. There is a good chance you simply need to update the `mojaloop-policy-generator.js` file and re-run the circleci workflow.
+
+For more information on anchore and anchore-cli, refer to:
+- [Anchore CLI](https://github.com/anchore/anchore-cli)
+- [Circle Orb Registry](https://circleci.com/orbs/registry/orb/anchore/anchore-engine)
+
diff --git a/audit-resolve.json b/audit-resolve.json
@@ -1,9 +1,44 @@
 {
   "decisions": {
-    "1482|@hapi/hapi": {
+    "1179|@mojaloop/central-services-shared>@mojaloop/event-sdk>grpc>node-pre-gyp>mkdirp>minimist": {
       "decision": "ignore",
-      "madeAt": 1582635042880,
-      "expiresAt": 1583239828928
+      "madeAt": 1588247839506,
+      "expiresAt": 1588852630163
+    },
+    "1179|@mojaloop/event-sdk>grpc>node-pre-gyp>mkdirp>minimist": {
+      "decision": "ignore",
+      "madeAt": 1588247839507,
+      "expiresAt": 1588852630163
+    },
+    "1179|@mojaloop/central-services-shared>@mojaloop/event-sdk>grpc>node-pre-gyp>tar>mkdirp>minimist": {
+      "decision": "ignore",
+      "madeAt": 1588247839507,
+      "expiresAt": 1588852630163
+    },
+    "1179|@mojaloop/event-sdk>grpc>node-pre-gyp>tar>mkdirp>minimist": {
+      "decision": "ignore",
+      "madeAt": 1588247839507,
+      "expiresAt": 1588852630163
+    },
+    "1179|@mojaloop/central-services-shared>@mojaloop/event-sdk>grpc>node-pre-gyp>rc>minimist": {
+      "decision": "ignore",
+      "madeAt": 1588247839507,
+      "expiresAt": 1588852630163
+    },
+    "1179|@mojaloop/event-sdk>grpc>node-pre-gyp>rc>minimist": {
+      "decision": "ignore",
+      "madeAt": 1588247839507,
+      "expiresAt": 1588852630163
+    },
+    "1500|npm-audit-resolver>yargs-unparser>yargs>yargs-parser": {
+      "decision": "ignore",
+      "madeAt": 1588760499425,
+      "expiresAt": 1589365290340
+    },
+    "1500|npm-audit-resolver>audit-resolve-core>yargs-parser": {
+      "decision": "ignore",
+      "madeAt": 1588760504403,
+      "expiresAt": 1589365290340
     }
   },
   "rules": {},