diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index ba36eb01..17a47a45 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -4,6 +4,8 @@ updates: - assignees: - "thomas-vogels" directory: "/" + ignore: + - dependency-name: "Sphinx" open-pull-requests-limit: 10 package-ecosystem: "pip" reviewers: diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index c163d212..d2cda041 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -18,11 +18,11 @@ jobs: - name: Checkout out code uses: actions/checkout@v3 - name: Set up Python 3.7 - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: 3.7 - name: Cache pip - uses: actions/cache@v3.0.2 + uses: actions/cache@v3.0.8 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('requirements-linters.txt') }} @@ -47,11 +47,11 @@ jobs: - name: Checkout out code uses: actions/checkout@v3 - name: Set up Python 3.7 - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: 3.7 - name: Cache pip - uses: actions/cache@v3.0.2 + uses: actions/cache@v3.0.8 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('requirements-linters.txt') }} diff --git a/.github/workflows/unittest.yaml b/.github/workflows/unittest.yaml index 17b647ff..2c3a6c10 100644 --- a/.github/workflows/unittest.yaml +++ b/.github/workflows/unittest.yaml @@ -16,11 +16,11 @@ jobs: - name: Checkout out code uses: actions/checkout@v3 - name: Set up Python 3.7 - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: 3.7 - name: Cache pip - uses: actions/cache@v3.0.2 + uses: actions/cache@v3.0.8 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('requirements-*.txt') }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bd53e89f..f91cccc1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,29 +39,29 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 22.8.0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 - rev: 4.0.1 + rev: 5.0.4 hooks: - id: flake8 additional_dependencies: - - "flake8-bugbear==22.4.25" - - "flake8-comprehensions==3.7.0" + - "flake8-bugbear==22.9.11" + - "flake8-comprehensions==3.10.0" - "flake8-docstrings==1.6.0" - "flake8-fixme==1.1.1" - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.950 + rev: v0.971 hooks: - id: mypy additional_dependencies: - - "mypy-boto3~=1.22" - - "types-PyYAML==6.0.7" - - "types-setuptools==57.4.14" - - "types-simplejson==3.17.6" - - "types-tabulate==0.8.8" - - "types-termcolor==1.1.4" + - "mypy-boto3~=1.24" + - "types-PyYAML==6.0.11" + - "types-setuptools==65.3.0" + - "types-simplejson==3.17.7" + - "types-tabulate==0.8.11" + - "types-termcolor==1.1.5" - repo: https://github.com/shellcheck-py/shellcheck-py rev: v0.7.2.1 hooks: diff --git a/bin/build_virtual_env b/bin/build_virtual_env index 45b74b8d..843af95c 100755 --- a/bin/build_virtual_env +++ b/bin/build_virtual_env @@ -23,6 +23,7 @@ source "$ARTHUR_VENV/bin/activate" echo "Creating or updating packages using pip..." python3 -m pip install --upgrade pip==20.3.4 --disable-pip-version-check python3 -m pip install --upgrade --requirement ./requirements-all.txt --disable-pip-version-check +python3 setup.py develop echo "To use this setup, you need to run:" echo " source '$ARTHUR_VENV/bin/activate'" diff --git a/bin/send_health_check.sh b/bin/send_health_check.sh index 2ee1c54a..a15df510 100755 --- a/bin/send_health_check.sh +++ b/bin/send_health_check.sh @@ -29,6 +29,7 @@ set -o errexit FILENAME="/tmp/redshift_etl/config/environment.sh" echo "Attempting to read configuration environment variables from '$FILENAME'" if [ -r "$FILENAME" ]; then + # shellcheck disable=SC1090 source "$FILENAME" else echo "Failed to find '$FILENAME'" diff --git a/cloudformation/dw_cluster.yaml b/cloudformation/dw_cluster.yaml index 9e818730..1038ad8a 100644 --- a/cloudformation/dw_cluster.yaml +++ b/cloudformation/dw_cluster.yaml @@ -74,6 +74,11 @@ Parameters: Type: String Default: "" + AdditionalClusterIAMRole6: + Description: (optional) ARN of an additional IAM role to associate with the Redshift cluster + Type: String + Default: "" + PreferredMaintenanceWindow: Description: (optional) Preferred maintenance window for the Redshift cluster Type: String @@ -102,6 +107,9 @@ Conditions: HasAdditionalRole5: !Not [ !Equals [ !Ref "AdditionalClusterIAMRole5", "" ] ] + HasAdditionalRole6: + !Not [ !Equals [ !Ref "AdditionalClusterIAMRole6", "" ] ] + Resources: @@ -121,12 +129,26 @@ Resources: Description: "Parameter group for Redshift cluster" ParameterGroupFamily: "redshift-1.0" Parameters: + - ParemeterName: "auto_analyze" + ParemeterValue: "true" + - ParemeterName: "auto_mv" + ParemeterValue: "true" + - ParemeterName: "datestyle " + ParemeterValue: "ISO, MDY" - ParameterName: "require_ssl" ParameterValue: "true" + - ParameterName: "enable_case_sensitive_identifier" + ParameterValue: "false" - ParameterName: "enable_user_activity_logging" ParameterValue: "true" + - ParameterName: "extra_float_digits" + ParameterValue: 0 + - ParameterName: "max_concurrency_scaling_clusters" + ParameterValue: 1 + - ParameterName: "search_path" + ParameterValue: "$user, public" - ParameterName: "statement_timeout" - ParameterValue: 7200000 + ParameterValue: 0 - ParameterName: "wlm_json_configuration" ParameterValue: !Sub "[{ \"query_concurrency\": ${QueryConcurrency} }]" Tags: @@ -162,6 +184,7 @@ Resources: - !If [ HasAdditionalRole3, !Ref "AdditionalClusterIAMRole3", !Ref "AWS::NoValue" ] - !If [ HasAdditionalRole4, !Ref "AdditionalClusterIAMRole4", !Ref "AWS::NoValue" ] - !If [ HasAdditionalRole5, !Ref "AdditionalClusterIAMRole5", !Ref "AWS::NoValue" ] + - !If [ HasAdditionalRole6, !Ref "AdditionalClusterIAMRole6", !Ref "AWS::NoValue" ] MasterUsername: !Ref MasterUsername MasterUserPassword: diff --git a/docker/.bash_profile b/docker/.bash_profile index 550d2272..a633a5ce 100644 --- a/docker/.bash_profile +++ b/docker/.bash_profile @@ -1,12 +1,16 @@ +#!/bin/bash + # This is the default .bash_profile in a container. # Get the aliases and functions if [ -f ~/.bashrc ]; then - . ~/.bashrc + # shellcheck disable=SC1090 + source ~/.bashrc fi # Initial greeting cat /opt/src/arthur-redshift-etl/etc/motd + echo -e "\nEnvironment settings:\n" arthur.py settings object_store.s3.* version echo diff --git a/docker/.bashrc b/docker/.bashrc index 8f13e01a..6e7b0197 100644 --- a/docker/.bashrc +++ b/docker/.bashrc @@ -1,8 +1,11 @@ +#!/bin/bash + # This is the default .bashrc file inside a container. # Source global definitions if [ -f /etc/bashrc ]; then - . /etc/bashrc + # shellcheck disable=SC1091 + source /etc/bashrc fi # Generally useful @@ -15,8 +18,9 @@ alias develop="( \cd /opt/src/arthur-redshift-etl && python setup.py develop )" PS1='(aws:$AWS_PROFILE, prefix:$ARTHUR_DEFAULT_PREFIX) \$ ' if [[ -z "$VIRTUAL_ENV" ]]; then + # shellcheck disable=SC1091 source /opt/local/redshift_etl/venv/bin/activate fi -# Commandline completion +# shellcheck disable=SC1091 source /opt/src/arthur-redshift-etl/etc/arthur_completion.sh diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 577d57dc..c34a4e95 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -19,6 +19,7 @@ set -o errexit -o nounset export PATH="/opt/local/redshift_etl/bin:$PATH" if [[ -r "/opt/local/redshift_etl/venv/bin/activate" ]]; then + # shellcheck disable=SC1091 source /opt/local/redshift_etl/venv/bin/activate # Using "--quiet" here to reduce the startup noise for "end users." diff --git a/docs/source/conf.py b/docs/source/conf.py index 46b1b246..06e42230 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -23,7 +23,7 @@ author = "Data Engineering at Harry's" # TODO(tom): Extract from setup.py -version = "1.59" +version = "1.64" # -- General configuration --------------------------------------------------- diff --git a/requirements-linters.txt b/requirements-linters.txt index d2d9dcf6..4c80ff33 100644 --- a/requirements-linters.txt +++ b/requirements-linters.txt @@ -1,18 +1,18 @@ # NOTE When changing versions here, be sure to keep .pre-commit-config.yaml consistent! -black==22.3.0 -flake8==4.0.1 -flake8-bugbear==22.4.25 -flake8-comprehensions==3.7.0 +black==22.8.0 +flake8==5.0.4 +flake8-bugbear==22.9.11 +flake8-comprehensions==3.10.0 flake8-docstrings==1.6.0 flake8-fixme==1.1.1 isort==5.10.1 -mypy==0.950 -mypy-boto3~=1.22 +mypy==0.971 +mypy-boto3~=1.24 mypy-extensions==0.4.3 pre-commit==2.17.0 pyupgrade==2.31.0 -types-PyYAML==6.0.7 -types-setuptools==57.4.14 -types-simplejson==3.17.6 -types-tabulate==0.8.8 -types-termcolor==1.1.4 +types-PyYAML==6.0.11 +types-setuptools==65.3.0 +types-simplejson==3.17.7 +types-tabulate==0.8.11 +types-termcolor==1.1.5 diff --git a/requirements.txt b/requirements.txt index 2664c5b6..16dcaa3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -# This is the set of packages need to run the code. All versions are pinned. -arrow==1.2.2 -boto3==1.22.4 -botocore~=1.25 +# This is the set of packages needed to run the code. All versions are pinned. +arrow==1.2.3 +boto3==1.23.10 +botocore~=1.26 docker==5.0.3 funcy==1.17 jmespath==0.10.0 @@ -10,7 +10,7 @@ pgpasslib==1.1.0 psycopg2-binary==2.9.3 PyYAML==6.0 simplejson==3.17.6 -tabulate==0.8.9 +tabulate==0.8.10 termcolor==1.1.0 -tqdm==4.64.0 +tqdm==4.64.1 watchtower==3.0.0 diff --git a/setup.py b/setup.py index bef351e5..eec24a01 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -ARTHUR_VERSION = "1.63.0" +ARTHUR_VERSION = "1.64.0" setup(