From 5a23193ad88f14c23162e18b6e62b7652ecf0093 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Mon, 16 Dec 2019 10:34:47 -0800 Subject: [PATCH] Use github actions, rather than travis (#229) * Use github actions, rather than travis * Pre-commit lint fixes on test directory * Run apt-get update prior to install * Create build directory for coverage * Temporarily lower coverage --- .github/workflows/pull_request.yml | 58 +++++++++++++++++ .github/workflows/push.yml | 65 +++++++++++++++++++ .nojekyll | 0 .pre-commit-config.yaml | 9 +++ .travis.yml | 26 -------- Gruntfile.js | 2 +- Makefile | 9 +-- .../public/modules/common/services/http.js | 2 +- .../controllers/ResourceHistoryCtrl.js | 6 +- .../modules/history/services/archive.js | 2 +- .../controllers/BlindCredentialDetailsCtrl.js | 3 +- .../controllers/CredentialDetailsCtrl.js | 9 +-- .../resources/controllers/ResourceCtrl.js | 8 +-- .../controllers/ServiceDetailsCtrl.js | 6 +- docker_push.sh | 18 ----- package.json | 2 +- setup.cfg | 4 +- tests/unit/confidant/authnz/authnz_test.py | 15 +++-- .../unit/confidant/encrypted_settings_test.py | 2 - .../confidant/services/keymanager_test.py | 7 +- 20 files changed, 168 insertions(+), 85 deletions(-) create mode 100644 .github/workflows/pull_request.yml create mode 100644 .github/workflows/push.yml create mode 100644 .nojekyll create mode 100644 .pre-commit-config.yaml delete mode 100644 .travis.yml delete mode 100755 docker_push.sh diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 00000000..7facaca7 --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,58 @@ +on: pull_request +jobs: + pre-commit: + runs-on: ubuntu-18.04 + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Setup python 3.6 + uses: actions/setup-python@v1 + with: + python-version: 3.6 + - name: Install pre-commit + run: pip install pre-commit + - name: Run pre-commit + run: pre-commit run --all-files + ## Uncomment once license tests are in and passing + #license-check: + # runs-on: ubuntu-18.04 + # steps: + # - name: Checkout + # uses: actions/checkout@v1 + # - name: Setup Ruby 2.x + # uses: actions/setup-ruby@v1 + # with: + # ruby-version: 2.x + # - name: Install license finder + # run: gem install license_finder + # - name: Setup python 3.6 + # uses: actions/setup-python@v1 + # with: + # python-version: 3.6 + # - name: Install apt dependencies + # run: sudo apt-get install -y python3-dev openssl libssl-dev gcc pkg-config libffi-dev libxml2-dev libxmlsec1-dev + # - name: Install dependencies + # run: pip install -r piptools_requirements3.txt && pip install -r requirements3.txt + # - name: Run license finder + # run: license_finder + test: + runs-on: ubuntu-18.04 + steps: + - name: Checkout + uses: actions/checkout@v1 + - uses: actions/setup-node@v1 + with: + node-version: '8.x' + - name: Setup python 3.6 + uses: actions/setup-python@v1 + with: + python-version: 3.6 + - name: Install apt dependencies + run: sudo apt-get update && sudo apt-get install -y python3-dev openssl libssl-dev gcc pkg-config libffi-dev libxml2-dev libxmlsec1-dev + - name: Install dependencies + run: pip install -r piptools_requirements3.txt && pip install -r requirements3.txt + - run: "npm install grunt-cli && npm install" + - name: Run python tests + run: make test + - name: Run frontend tests + run: grunt test diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 00000000..f4c20393 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,65 @@ +name: Build and Deploy +on: + push: + branches: + - master + tags: + - '*' +jobs: + build-and-deploy-docs: + name: Build and publish docs + runs-on: ubuntu-18.04 + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Setup Ruby + uses: actions/setup-ruby@v1 + with: + ruby-version: '2.6.x' + - name: Install bundler deps + run: | + cd docs + gem install bundler + bundle install --jobs 4 --retry 3 + - name: Build docs + run: cd docs && bundler exec middleman build && touch build/.nojekyll + - name: Deploy docs + uses: JamesIves/github-pages-deploy-action@releases/v3 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BASE_BRANCH: master # The branch the action should deploy from. + BRANCH: gh-pages # The branch the action should deploy to. + FOLDER: docs/build # The folder the action should deploy. + build-and-publish-python-module: + name: Build and publish python module to pypi + runs-on: ubuntu-18.04 + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Setup python 3.6 + uses: actions/setup-python@v1 + with: + python-version: 3.6 + - name: Add wheel dependency + run: pip install wheel + - name: Generate dist + run: python setup.py sdist bdist_wheel + - name: Publish to PyPI + if: startsWith(github.event.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.pypi_password }} + build-and-publish-docker-image: + name: Build and publish docker image + runs-on: ubuntu-18.04 + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Publish to Registry + uses: elgohr/Publish-Docker-Github-Action@2.8 + with: + name: lyft/confidant + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + tag_names: true diff --git a/.nojekyll b/.nojekyll new file mode 100644 index 00000000..e69de29b diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..a94479a6 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,9 @@ +exclude: '^docs/.*$' +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.1.0 + hooks: + - id: flake8 + additional_dependencies: + - flake8==3.3.0 + - flake8-tidy-imports==1.0.6 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 937cf73f..00000000 --- a/.travis.yml +++ /dev/null @@ -1,26 +0,0 @@ -dist: bionic -language: python -env: - - REPO=lyft/confidant -sudo: required -services: - - docker -before_install: - - docker build -f Dockerfile -t $REPO:$TRAVIS_COMMIT . - - docker run -v $PWD/confidant/dist:/tmp/dist $REPO:$TRAVIS_COMMIT /bin/sh -c "cp -r /srv/confidant/confidant/dist/. /tmp/dist/." -install: - - gem install travis --no-document -script: - - travis lint .travis.yml --skip-completion-check - - docker run $REPO:$TRAVIS_COMMIT /bin/sh -c "make test" -after_success: - - ./docker_push.sh -deploy: - provider: pypi - user: lyftpypi - password: - secure: J56TAJm3sarDyy5yvAX3m/4NOJwF4DlJx/Io0MJbiWSLu1/RzH3akXOtFk37GL+CqnK+ffFMPcjKGX+UO9Gc7+cMX7Z58nk6aeDF+sf2Irh8arZVhNenLQVYuVxtlfqOfagTJEyVAjFU8T9Iya5f2VontkMdrraXdrNmNRTJfbgp/sWKFnR2Z2vqS66aQ2hOdviYJVzQ/Vr5cGq/epiZ8g4BG42pXHqYgA73AZELzhdavNnLSP9UGZZ49dW9ek+PDo4iFcVSfJ7BkICJelC4nxTv7SYSMOzhDbDjIB7Il7XtrrkvVetWAyCtoF3CEDTloKV0KLtKkHkYIF8slaMTETQhOEgnv0jzpvTtJHcZ60z05njyPMt3UGtvlGYhDgg9LvO5saj/cW2uI++CJ/QInD7wP64UlTFraWasi02Ir190wKnOPRlBD24xF0udvlHqIOE+wVvA4tZBXgW6cGZ3szYFSsfKTZAmSMT7ey3tiudhMh4bQ3riduJPULpCuIaEQazsbiUmTKSy2SjBUpKcpDNKK8062NnVWjaI/1wsV/I37GTpAWOtLdxp81BPBwM4XyvnNNBUxcuExgE36YG2c01+MdGPqBFf++IyfcZPY8vtIggE9rvg8tbWbiCFFI7It1pVNZufYm5o64RZVo7GJqL3/DR0PfxC8xSidW6E9OU= - distributions: sdist bdist_wheel - on: - tags: true - repo: lyft/confidant diff --git a/Gruntfile.js b/Gruntfile.js index a760869a..79125e54 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -45,7 +45,7 @@ module.exports = function (grunt) { // Make sure code styles are up to par and there are no obvious mistakes jshint: { options: { - jshintrc: '.jshintrc', + jshintrc: '<%= baseDir %>.jshintrc', reporter: require('jshint-stylish') }, all: { diff --git a/Makefile b/Makefile index 55c47b75..bdf61af3 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,8 @@ # bash needed for pipefail SHELL := /bin/bash -test: test_lint test_unit - -test_lint: - mkdir -p build - set -o pipefail; flake8 | sed "s#^\./##" > build/flake8.txt || (cat build/flake8.txt && exit 1) +test: test_unit test_unit: - nosetests --with-path=confidant tests/unit + mkdir -p build + nosetests --with-coverage --with-path=confidant tests/unit diff --git a/confidant/public/modules/common/services/http.js b/confidant/public/modules/common/services/http.js index 0014245a..cb5ab3f8 100644 --- a/confidant/public/modules/common/services/http.js +++ b/confidant/public/modules/common/services/http.js @@ -80,7 +80,7 @@ responseError: function(rejection) { var status = rejection.status; finish(); - if (status == 401) { + if (status === 401) { window.location.href = 'loggedout'; } return $q.reject(rejection); diff --git a/confidant/public/modules/history/controllers/ResourceHistoryCtrl.js b/confidant/public/modules/history/controllers/ResourceHistoryCtrl.js index 40676dc4..d070382d 100644 --- a/confidant/public/modules/history/controllers/ResourceHistoryCtrl.js +++ b/confidant/public/modules/history/controllers/ResourceHistoryCtrl.js @@ -49,11 +49,11 @@ $scope.resourceTypeFilter = function(field) { return function(resource) { - if (resource[field] == 'credential' && $scope.showCredentials) { + if (resource[field] === 'credential' && $scope.showCredentials) { return true; - } else if (resource[field] == 'blind_credential' && $scope.showBlindCredentials) { + } else if (resource[field] === 'blind_credential' && $scope.showBlindCredentials) { return true; - } else if (resource[field] == 'service' && $scope.showServices) { + } else if (resource[field] === 'service' && $scope.showServices) { return true; } return false; diff --git a/confidant/public/modules/history/services/archive.js b/confidant/public/modules/history/services/archive.js index 345a61b5..6e63a13b 100644 --- a/confidant/public/modules/history/services/archive.js +++ b/confidant/public/modules/history/services/archive.js @@ -34,7 +34,7 @@ for (var i = credentialArchive.length; i--;) { credentialArchive[i].type = 'credential'; } - for (var i = blindCredentialArchive.length; i--;) { + for (i = blindCredentialArchive.length; i--;) { blindCredentialArchive[i].type = 'blind_credential'; } for (i = serviceArchive.length; i--;) { diff --git a/confidant/public/modules/resources/controllers/BlindCredentialDetailsCtrl.js b/confidant/public/modules/resources/controllers/BlindCredentialDetailsCtrl.js index 04f6ec16..025eccbd 100644 --- a/confidant/public/modules/resources/controllers/BlindCredentialDetailsCtrl.js +++ b/confidant/public/modules/resources/controllers/BlindCredentialDetailsCtrl.js @@ -19,12 +19,11 @@ 'blindcredentials.credential', 'blindcredentials.services', function ($scope, $stateParams, $q, $log, $filter, $location, BlindCredential, BlindCredentialServices) { - var credentialCopy = null; $scope.$log = $log; if ($stateParams.blindCredentialId) { BlindCredentialServices.get({'id': $stateParams.blindCredentialId}).$promise.then(function(blindCredentialServices) { - $scope.blindCredentialServices = blindCredentialServices['services']; + $scope.blindCredentialServices = blindCredentialServices.services; }); BlindCredential.get({'id': $stateParams.blindCredentialId}).$promise.then(function(blindCredential) { diff --git a/confidant/public/modules/resources/controllers/CredentialDetailsCtrl.js b/confidant/public/modules/resources/controllers/CredentialDetailsCtrl.js index fdb30f31..dcb112a7 100644 --- a/confidant/public/modules/resources/controllers/CredentialDetailsCtrl.js +++ b/confidant/public/modules/resources/controllers/CredentialDetailsCtrl.js @@ -42,7 +42,8 @@ 'credentials.services', 'credentials.valueGenerator', function ($scope, $stateParams, $q, $log, $filter, $location, Credential, Credentials, CredentialServices, ValueGenerator) { - var credentialCopy = null; + var credentialCopy = null, + deferred = $q.defer(); $scope.$log = $log; $scope.saveError = ''; $scope.getError = ''; @@ -54,7 +55,7 @@ if ($stateParams.credentialId) { CredentialServices.get({'id': $stateParams.credentialId}).$promise.then(function(credentialServices) { - $scope.credentialServices = credentialServices['services']; + $scope.credentialServices = credentialServices.services; }); Credential.get({'id': $stateParams.credentialId}).$promise.then(function(credential) { @@ -192,7 +193,7 @@ } // Ensure metadata keys are unique and transform them // into key/value dict. - for (var i = $scope.credential.mungedMetadata.length; i--;) { + for (i = $scope.credential.mungedMetadata.length; i--;) { var metadataItem = $scope.credential.mungedMetadata[i]; if (metadataItem.isDeleted) { $scope.credential.mungedMetadata.splice(i, 1); @@ -224,7 +225,7 @@ newCredential.mungedMetadata = _metadata; $scope.credential = newCredential; if (credentialCopy.name !== $scope.credential.name || - credentialCopy.enabled != $scope.credential.enabled) { + credentialCopy.enabled !== $scope.credential.enabled) { $scope.$emit('updateCredentialList'); } credentialCopy = angular.copy(newCredential); diff --git a/confidant/public/modules/resources/controllers/ResourceCtrl.js b/confidant/public/modules/resources/controllers/ResourceCtrl.js index 638044c2..ce289473 100644 --- a/confidant/public/modules/resources/controllers/ResourceCtrl.js +++ b/confidant/public/modules/resources/controllers/ResourceCtrl.js @@ -53,12 +53,12 @@ }; $scope.resourceTypeFilter = function(field) { - return function(resource) { - if (field == 'credential' && $scope.showCredential) { + return function() { + if (field === 'credential' && $scope.showCredential) { return true; - } else if (field == 'blind-credential' && $scope.showBlindCredential) { + } else if (field === 'blind-credential' && $scope.showBlindCredential) { return true; - } else if (field == 'service' && $scope.showService) { + } else if (field === 'service' && $scope.showService) { return true; } return false; diff --git a/confidant/public/modules/resources/controllers/ServiceDetailsCtrl.js b/confidant/public/modules/resources/controllers/ServiceDetailsCtrl.js index 1555fc2c..0e1a2712 100644 --- a/confidant/public/modules/resources/controllers/ServiceDetailsCtrl.js +++ b/confidant/public/modules/resources/controllers/ServiceDetailsCtrl.js @@ -143,7 +143,7 @@ $scope.service.credentials.splice(i, 1); } } - for (var i = $scope.service.blind_credentials.length; i--;) { + for (i = $scope.service.blind_credentials.length; i--;) { var blind_credential = $scope.service.blind_credentials[i]; if (blind_credential.isDeleted) { delete blind_credential.isDeleted; @@ -196,7 +196,7 @@ deferred.resolve(); } return deferred.promise; - } + }; $scope.saveService = function() { var _service = {}, @@ -221,7 +221,7 @@ } _service.credentials.push(credential.id); } - for (var i = $scope.service.blind_credentials.length; i--;) { + for (i = $scope.service.blind_credentials.length; i--;) { var blind_credential = $scope.service.blind_credentials[i]; if (blind_credential.isDeleted) { $scope.service.blind_credentials.splice(i, 1); diff --git a/docker_push.sh b/docker_push.sh deleted file mode 100755 index 3875cbc3..00000000 --- a/docker_push.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -if [ "$TRAVIS_PULL_REQUEST" == "false" -a -n "$TRAVIS_TAG" ] -then - docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD - export TAG="$TRAVIS_TAG" - echo "TAG is $TAG" - docker tag $TRAVIS_REPO_SLUG:$TRAVIS_COMMIT $REPO:$TAG - docker push $TRAVIS_REPO_SLUG:$TAG -elif [ "$TRAVIS_PULL_REQUEST" == "false" -a "$TRAVIS_BRANCH" == "master" ] -then - docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD - export TAG="latest" - echo "TAG is $TAG" - docker tag $TRAVIS_REPO_SLUG:$TRAVIS_COMMIT $REPO:$TAG - docker push $TRAVIS_REPO_SLUG:$TAG -else - echo 'Ignoring PR branch for docker push.' -fi diff --git a/package.json b/package.json index 65e823cc..02e79545 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "grunt-contrib-copy": "^0.5.0", "grunt-contrib-cssmin": "^0.9.0", "grunt-contrib-htmlmin": "^0.3.0", - "grunt-contrib-jshint": "^0.10.0", + "grunt-contrib-jshint": "^2.1.0", "grunt-contrib-symlink": "^0.3.0", "grunt-contrib-uglify": "^0.4.0", "grunt-contrib-watch": "^0.6.1", diff --git a/setup.cfg b/setup.cfg index a4c51711..ac896199 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,7 +13,7 @@ xunit-file = build/nosetests.xml cover-package = confidant cover-xml = 1 cover-xml-file = build/coverage.xml -cover-min-percentage = 39 +cover-min-percentage = 37 [flake8] # The jenkins violations plugin can read the pylint format. @@ -25,7 +25,7 @@ max-line-length = 80 # venv/: # third party libraries are all stored in venv - so we don't want to # check them for style issues. -exclude = .git,__pycache__,venv,tests/,.ropeproject,confidant-testing.py +exclude = .git,__pycache__,venv,tests/,.ropeproject [pep8] max-line-length = 80 diff --git a/tests/unit/confidant/authnz/authnz_test.py b/tests/unit/confidant/authnz/authnz_test.py index 71b16ef5..4a0f42e4 100644 --- a/tests/unit/confidant/authnz/authnz_test.py +++ b/tests/unit/confidant/authnz/authnz_test.py @@ -8,9 +8,9 @@ settings.encrypted_settings.secret_string = {} settings.encrypted_settings.decrypted_secrets = {'SESSION_SECRET': 'TEST_KEY'} -import confidant.routes -from confidant.app import app -from confidant import authnz +import confidant.routes # noqa:E402,F401 +from confidant.app import app # noqa:E402 +from confidant import authnz # noqa:E402 class AuthnzTest(unittest.TestCase): @@ -159,16 +159,19 @@ def test_will_extract_from_request(self): with patch('confidant.authnz.userauth.request') as request_mock: request_mock.headers = { app.config['HEADER_AUTH_USERNAME_HEADER']: 'unittestuser', - app.config['HEADER_AUTH_EMAIL_HEADER']: 'unittestuser@example.com', + app.config['HEADER_AUTH_EMAIL_HEADER']: 'unittestuser@example.com', # noqa:E501 } - self.assertEqual(authnz.get_logged_in_user(), 'unittestuser@example.com') + self.assertEqual( + authnz.get_logged_in_user(), + 'unittestuser@example.com' + ) def test_will_log_in(self): with app.test_request_context('/fake'): with patch('confidant.authnz.userauth.request') as request_mock: request_mock.headers = { app.config['HEADER_AUTH_USERNAME_HEADER']: 'unittestuser', - app.config['HEADER_AUTH_EMAIL_HEADER']: 'unittestuser@example.com', + app.config['HEADER_AUTH_EMAIL_HEADER']: 'unittestuser@example.com', # noqa:E501 } resp = authnz.user_mod.log_in() diff --git a/tests/unit/confidant/encrypted_settings_test.py b/tests/unit/confidant/encrypted_settings_test.py index 4132c51e..80f62b51 100644 --- a/tests/unit/confidant/encrypted_settings_test.py +++ b/tests/unit/confidant/encrypted_settings_test.py @@ -1,8 +1,6 @@ import unittest from mock import patch -from mock import Mock -from confidant import settings from confidant.encrypted_settings import EncryptedSettings diff --git a/tests/unit/confidant/services/keymanager_test.py b/tests/unit/confidant/services/keymanager_test.py index 781e97b7..7718a092 100644 --- a/tests/unit/confidant/services/keymanager_test.py +++ b/tests/unit/confidant/services/keymanager_test.py @@ -1,16 +1,13 @@ import unittest -import datetime -import json from mock import patch -from mock import MagicMock # Prevent call to KMS during tests from confidant import settings settings.encrypted_settings.secret_string = {} -from confidant.services import keymanager -from confidant.app import app +from confidant.services import keymanager # noqa:E402 +from confidant.app import app # noqa:E402 class KeyManagerTest(unittest.TestCase):