Merge pull request #1 from sanger/first_commit

DPL-361-3 Create python library

.github/workflows/automated_release.yml

@@ -0,0 +1,36 @@
+name: Automated build, test, release and push
+
+on:
+  push:
+    branches:
+      - develop
+      - master
+
+env:
+  IMAGE_NAME: ${{ github.repository }}/${{ github.event.repository.name }}
+
+jobs:
+  build_test_release_push:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: nelonoel/[email protected]
+
+      # Create a release tag based on the branch name and .release-version file
+      - name: Set release tag
+        # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
+        # On the develop branch this might create RELEASE_VERSION=2.4.6-987654321-develop
+        # On the master branch this would then only create RELEASE_VERSION=2.4.6
+        run: echo "RELEASE_VERSION=$(printf -- '%s%s\n' $(cat .release-version) $([ ${BRANCH_NAME} = "develop" ] && printf -- '-%s-develop' ${GITHUB_RUN_ID} || echo ""))" >> $GITHUB_ENV
+
+      # Create a GitHub release with the release asset as an artifact
+      - name: Create release and upload release.tar.gz
+        uses: ncipollo/[email protected]
+        with:
+          name: ${{ env.RELEASE_VERSION }}
+          tag: v${{ env.RELEASE_VERSION }}
+          prerelease: ${{ !(github.ref == 'refs/heads/master') }}
+          commit: ${{ github.sha }}
+          artifacts: dist/*.tar.gz
+          artifactErrorsFailBuild: true

.github/workflows/check_release_version.yml

@@ -0,0 +1,27 @@
+# Checks that the .release-version file has been updated
+name: Check release version
+
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Get specific changed files
+        id: changed-files-specific
+        uses: tj-actions/changed-files@v23
+        with:
+          files: |
+            .release-version
+          
+      - name: Run step looking for change in the release version
+        run: >-
+          if ! "${{ steps.changed-files-specific.outputs.any_changed }}"; then
+            echo "Please change the release version number"
+            exit 1;
+          fi

.github/workflows/ci.yml

@@ -0,0 +1,111 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - develop-*
+      - develop
+      - master
+  pull_request:
+    branches:
+      - develop-*
+      - develop
+      - master
+
+jobs:
+  black:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/Pipfile') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Install pipenv
+        run: |
+          pip install pipenv
+      - name: Install dependencies
+        run: |
+          pipenv sync --dev --system
+      - name: Check format with black
+        run: |
+          # stop the build if there are black formatting errors
+          python -m black --check .
+  flake8:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/Pipfile') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Install pipenv
+        run: |
+          pip install pipenv
+      - name: Install dependencies
+        run: |
+          pipenv sync --dev --system
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8
+  mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/Pipfile') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Install pipenv
+        run: |
+          pip install pipenv
+      - name: Install dependencies
+        run: |
+          pipenv sync --dev --system
+      - name: Run mypy
+        run: |
+          python -m mypy .
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/Pipfile') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Install pipenv
+        run: |
+          pip install pipenv
+      - name: Install dependencies
+        run: |
+          pipenv sync --dev --system
+      - name: Test with pytest
+        run: |
+          python -m pytest -vx
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,72 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+# ******** NOTE ********
+
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ develop-*, develop, master ]
+    paths-ignore:
+      - "README.md"
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ develop-*, develop ]
+    paths-ignore:
+      - "README.md"
+  schedule:
+    - cron: '40 23 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more...
+        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    # - name: Autobuild
+    #   uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.release-version

@@ -0,0 +1 @@
+0.1.0

Dockerfile

@@ -0,0 +1,44 @@
+# Use alpine for a smaller image size and install only the required packages
+FROM python:3.8-slim-buster
+
+# > Setting PYTHONUNBUFFERED to a non empty value ensures that the python output is sent straight to
+# > terminal (e.g. your container log) without being first buffered and that you can see the output
+# > of your application (e.g. django logs) in real time.
+# https://stackoverflow.com/a/59812588
+ENV PYTHONUNBUFFERED 1
+
+# Install required libs
+RUN apt-get update && \
+    apt-get install -y \
+    build-essential \
+    curl \
+    netcat \
+    unixodbc-dev
+
+# Install the package manager - pipenv
+RUN pip install --upgrade pip && \
+    pip install --no-cache-dir pipenv
+
+# Change the working directory for all proceeding operations
+#   https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#workdir
+WORKDIR /code
+
+# "items (files, directories) that do not require ADD’s tar auto-extraction capability, you should always use COPY."
+#   https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#add-or-copy
+COPY Pipfile .
+COPY Pipfile.lock .
+
+# Install both default and dev packages so that we can run the tests against this image
+RUN pipenv sync --dev --system && \
+    pipenv --clear
+
+# Copy all the source to the image
+COPY . .
+
+# "The best use for ENTRYPOINT is to set the image’s main command, allowing that image to be run as though it was that
+#   command (and then use CMD as the default flags)."
+#   https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#entrypoint
+
+# https://docs.docker.com/engine/reference/builder/#healthcheck
+#HEALTHCHECK --interval=1m --timeout=3s \
+#    CMD curl -f http://localhost:8000/health || exit 1

Pipfile

@@ -0,0 +1,41 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+black = "*"
+coverage = {extras = ["toml"],version = "*"}
+flake8 = "*"
+flake8-bugbear = "*"
+mypy = "*"
+pika-stubs = "*"
+pytest = "*"
+pytest-cov = "*"
+pytest-freezegun = "*"
+types-python-dateutil = "*"
+responses = "*"
+types-requests = "*"
+build = "*"
+hatchling = "*"
+
+[packages]
+colorlog = "~=6.6"
+fastavro = "~=1.5"
+more-itertools = "~=8.13"
+pika = "~=1.3"
+python-dotenv = "~=0.20"
+requests = "~=2.28"
+slackclient = "~=2.9"
+
+[requires]
+python_version = "3.8"
+
+[pipenv]
+allow_prereleases = true
+
+[scripts]
+black = 'black'
+mypy = 'mypy'
+flake8 = 'flake8'
+test = 'python -m pytest -vx'