From 1d586f1c64a1076431786f2641aa3e7da99ebda4 Mon Sep 17 00:00:00 2001 From: Marcos Schroh Date: Tue, 18 Feb 2020 11:14:48 +0100 Subject: [PATCH] feat: kubernetes resources added. linting fixed (#46) --- .travis.yml | 14 +- README.md | 37 ++-- cookiecutter.json | 2 +- requirements.txt | 22 +-- scripts/lint | 9 + scripts/lint.sh | 13 -- scripts/{publish.sh => publish} | 2 +- scripts/test | 8 + scripts/test.sh | 13 -- scripts/{test_docker.sh => test_docker} | 10 +- setup.cfg | 6 +- setup.py | 6 +- tests/test_cookiecutter_generation.py | 8 +- {{cookiecutter.project_slug}}/.gitignore | 170 +----------------- {{cookiecutter.project_slug}}/.travis.yml | 2 +- {{cookiecutter.project_slug}}/README.md | 4 +- {{cookiecutter.project_slug}}/k8s/.helmignore | 21 +++ {{cookiecutter.project_slug}}/k8s/Chart.yaml | 5 + .../k8s/templates/_helpers.tpl | 33 ++++ .../k8s/templates/configmap.yaml | 17 ++ .../k8s/templates/deployment.yaml | 53 ++++++ .../k8s/templates/ingress.yaml | 21 +++ .../k8s/templates/service.yaml | 15 ++ {{cookiecutter.project_slug}}/k8s/values.yaml | 50 ++++++ .../k8s/values/prod.yaml | 10 ++ .../scripts/README.md | 1 + {{cookiecutter.project_slug}}/scripts/clean | 17 ++ {{cookiecutter.project_slug}}/scripts/lint | 9 + {{cookiecutter.project_slug}}/scripts/test | 10 ++ {{cookiecutter.project_slug}}/scripts/test.sh | 14 -- {{cookiecutter.project_slug}}/setup.cfg | 8 +- {{cookiecutter.project_slug}}/setup.py | 16 +- .../test_requirements.txt | 18 +- .../tests/conftest.py | 2 +- .../tests/test_page_views.py | 8 +- .../codecs/codec.py | 2 +- .../{{cookiecutter.project_slug}}/settings.py | 18 +- 37 files changed, 371 insertions(+), 303 deletions(-) create mode 100755 scripts/lint delete mode 100755 scripts/lint.sh rename scripts/{publish.sh => publish} (95%) create mode 100755 scripts/test delete mode 100755 scripts/test.sh rename scripts/{test_docker.sh => test_docker} (79%) create mode 100644 {{cookiecutter.project_slug}}/k8s/.helmignore create mode 100644 {{cookiecutter.project_slug}}/k8s/Chart.yaml create mode 100644 {{cookiecutter.project_slug}}/k8s/templates/_helpers.tpl create mode 100644 {{cookiecutter.project_slug}}/k8s/templates/configmap.yaml create mode 100644 {{cookiecutter.project_slug}}/k8s/templates/deployment.yaml create mode 100644 {{cookiecutter.project_slug}}/k8s/templates/ingress.yaml create mode 100644 {{cookiecutter.project_slug}}/k8s/templates/service.yaml create mode 100644 {{cookiecutter.project_slug}}/k8s/values.yaml create mode 100644 {{cookiecutter.project_slug}}/k8s/values/prod.yaml create mode 100755 {{cookiecutter.project_slug}}/scripts/clean create mode 100755 {{cookiecutter.project_slug}}/scripts/lint create mode 100755 {{cookiecutter.project_slug}}/scripts/test delete mode 100755 {{cookiecutter.project_slug}}/scripts/test.sh diff --git a/.travis.yml b/.travis.yml index 6454f73..3b91be2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,14 +15,8 @@ install: pip install -r requirements.txt matrix: include: - - name: test python 3.6 - python: 3.6 - script: ./scripts/test.sh - - name: test python 3.7 + - name: Test python: 3.7 - script: ./scripts/test.sh - - name: lint - script: ./scripts/lint.sh - - name: Docker image - script: ./scripts/test_docker.sh - + script: ./scripts/test + - name: Test Docker image + script: ./scripts/test_docker diff --git a/README.md b/README.md index 4decf3b..b492d34 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,6 @@ Cookiecutter Faust [![Build Status](https://travis-ci.org/marcosschroh/cookiecutter-faust.svg?branch=master)](https://travis-ci.org/marcosschroh/cookiecutter-faust) [![GitHub license](https://img.shields.io/github/license/marcosschroh/cookiecutter-faust.svg)](https://github.com/marcosschroh/cookiecutter-faust/blob/feature/add-license-and-remove-network-after-clean/LICENSE) - Table of Contents ----------------- @@ -18,16 +17,15 @@ Table of Contents Features -------- -* For Faust 1.9.0 -* Python 3.7 -* Docker and docker-compose support -* Useful commands included in Makefile -* project skeleton is defined as a medium/large project according to [faust layout](https://faust.readthedocs.io/en/latest/userguide/application.html#projects-and-directory-layout) -* The `setup.py` has the entrypoint to resolve the [entrypoint problem](https://faust.readthedocs.io/en/latest/userguide/application.html#problem-entrypoint) - +- Python 3.7+ +- Docker and docker-compose for development +- Useful commands included in Makefile +- Project skeleton is defined as a medium/large project according to [faust layout](https://faust.readthedocs.io/en/latest/userguide/application.html#projects-and-directory-layout) +- The `setup.py` has the entrypoint to resolve the [entrypoint problem](https://faust.readthedocs.io/en/latest/userguide/application.html#problem-entrypoint) +- Kubernetes manifests included Usage ------- +----- Let's pretend you want to create a Faust project called "super faust". @@ -71,7 +69,7 @@ include_page_view_tutorial [n]: y worker_port [6066]: kafka_server_environment_variable [KAFKA_BOOTSTRAP_SERVER]: include_codec_example [y]: -Select faust_loglevel: +Select log_level: 1 - CRITICAL 2 - ERROR 3 - WARNING @@ -126,7 +124,6 @@ git push -u origin master Now take a look at your repo. Don't forget to carefully look at the generated README. Awesome, right? - Useful Commands --------------- @@ -140,7 +137,6 @@ Useful Commands | `make send-page-view-event payload='{a payload}'`| Send event to a page view application | -- | `make send-page-view-event payload='{"id": "foo", "user": "bar"}'` | | `make list-agents`| List faust agents| --- | | - Settings -------- @@ -174,7 +170,7 @@ LOGGING = { }, 'handlers': { 'console': { - 'level': '{{cookiecutter.faust_loglevel}}', + 'level': '{{cookiecutter.log_level}}', 'class': 'logging.StreamHandler', 'formatter': 'default', }, @@ -182,7 +178,7 @@ LOGGING = { 'loggers': { 'your_project_slug': { 'handlers': ['console'], - 'level': '{{cookiecutter.faust_loglevel}}', + 'level': '{{cookiecutter.log_level}}', }, }, } @@ -215,10 +211,17 @@ Useful `ENVIRONMENT` variables that you may change: | SCHEMA_REGISTRY_SERVER_PORT | Schema registry server port | `8081` | | SCHEMA_REGISTRY_URL | Schema Registry Server url | `http://schema-registry:8081` | +Development +----------- + +Run tests: + +```bash +./scripts/test +``` -Run tests ----------- +Run code linting ```bash -./scripts/test.sh +./scripts/lint ``` diff --git a/cookiecutter.json b/cookiecutter.json index 091d771..cfe427c 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -20,7 +20,7 @@ "worker_port": 6066, "kafka_server_environment_variable": "KAFKA_BOOTSTRAP_SERVER", "include_codec_example": "y", - "faust_loglevel": [ + "log_level": [ "INFO", "CRITICAL", "ERROR", diff --git a/requirements.txt b/requirements.txt index 660c5a0..bcec7b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,17 @@ -cookiecutter==1.6.0 -sh==1.12.14 -binaryornot==0.4.4 +cookiecutter +sh +binaryornot # Code quality # ------------------------------------------------------------------------------ -black==19.10b0 -flake8==3.7.9 -isort==4.3.21 -mypy==0.761 +black +flake8 +isort +mypy # Testing # ------------------------------------------------------------------------------ -pytest==5.3.2 -pytest_cases==1.6.2 -pytest-cookies==0.3.0 -pytest-asyncio==0.10.0 +pytest +pytest_cases +pytest-cookies +pytest-asyncio diff --git a/scripts/lint b/scripts/lint new file mode 100755 index 0000000..94d6c9a --- /dev/null +++ b/scripts/lint @@ -0,0 +1,9 @@ +#!/bin/sh -e + +export PREFIX="" +if [ -d 'venv' ] ; then + export PREFIX="venv/bin/" +fi + +${PREFIX}isort --multi-line=3 --trailing-comma --force-grid-wrap=0 --combine-as --line-width 120 --recursive --apply src tests +${PREFIX}black src tests diff --git a/scripts/lint.sh b/scripts/lint.sh deleted file mode 100755 index 37710c4..0000000 --- a/scripts/lint.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -export PREFIX="" -if [ -d 'venv' ] ; then - export PREFIX="venv/bin/" -fi - -set -x - -${PREFIX}flake8 hooks tests setup.py -${PREFIX}black --check hooks tests setup.py -# ${PREFIX}autoflake --in-place --recursive schema_registry tests setup.py -${PREFIX}isort --multi-line=3 --trailing-comma --force-grid-wrap=0 --combine-as --line-width 100 -rc . diff --git a/scripts/publish.sh b/scripts/publish similarity index 95% rename from scripts/publish.sh rename to scripts/publish index 14e2f6d..03a8a50 100755 --- a/scripts/publish.sh +++ b/scripts/publish @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e VERSION=`cat setup.py | grep '__version__ =' | sed 's/__version__ = //' | sed 's/"//g'` diff --git a/scripts/test b/scripts/test new file mode 100755 index 0000000..5beaec2 --- /dev/null +++ b/scripts/test @@ -0,0 +1,8 @@ +#!/bin/sh -e + +export PREFIX="" +if [ -d 'venv' ] ; then + export PREFIX="venv/bin/" +fi + +${PREFIX}pytest ${1-"./tests"} diff --git a/scripts/test.sh b/scripts/test.sh deleted file mode 100755 index cfe31c7..0000000 --- a/scripts/test.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -e - -export PREFIX="" -if [ -d 'venv' ] ; then - export PREFIX="venv/bin/" -fi - -export VERSION_SCRIPT="import sys; print('%s.%s' % sys.version_info[0:2])" -export PYTHON_VERSION=`python -c "$VERSION_SCRIPT"` - -set -x - -PYTHONPATH=. ${PREFIX}pytest ${1-"./tests"} diff --git a/scripts/test_docker.sh b/scripts/test_docker similarity index 79% rename from scripts/test_docker.sh rename to scripts/test_docker index f63d5ab..0300370 100755 --- a/scripts/test_docker.sh +++ b/scripts/test_docker @@ -1,11 +1,10 @@ -#!/bin/sh -# sh tests/test_docker.sh +#!/bin/sh -e set -o errexit project_name="my_awesome_faust_project" # install test requirements -pip install faust==1.9.0 simple-settings==0.16.0 +pip install faust simple-settings pip install -r ./requirements.txt # create a cache directory @@ -18,9 +17,10 @@ cd ${project_name} docker-compose stop echo yes | docker-compose rm -docker network rm ${project_name} | true -./scripts/test.sh +docker network rm ${project_name} || true + +./scripts/test # Listing the agents to prove that the project generated is working docker-compose run -e SIMPLE_SETTINGS=${project_name}.settings ${project_name} faust -A ${project_name}.app agents diff --git a/setup.cfg b/setup.cfg index f4fde7a..68be835 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,11 @@ [flake8] -max-line-length = 100 +max-line-length = 120 exclude = .tox,.git,docs +select = B,E,F,W,C,W504,B902,B903,B950 +ignore = E203,E231,E902,W503 [pycodestyle] -max-line-length = 100 +max-line-length = 120 exclude = .tox,.git,docs [mypy] diff --git a/setup.py b/setup.py index f66d614..a9aca40 100644 --- a/setup.py +++ b/setup.py @@ -21,12 +21,12 @@ download_url="", packages=[], include_package_data=True, - license="GPLv3", + license="MIT", classifiers=[ - "Framework :: Faust :: 1.9.0", + "Framework :: Faust", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Topic :: Software Development", ], keywords=( diff --git a/tests/test_cookiecutter_generation.py b/tests/test_cookiecutter_generation.py index c13fe63..83e4627 100644 --- a/tests/test_cookiecutter_generation.py +++ b/tests/test_cookiecutter_generation.py @@ -10,7 +10,7 @@ RE_OBJ = re.compile(PATTERN) YN_CHOICES = ["y", "n"] -FAUST_LOGLEVEL = ["CRITICAL", "ERROR", ] +log_level = ["CRITICAL", "ERROR", ] CI_PROVIDERS = ["travis", "none", ] WORKER_PORT = [6066, 8000, 8080] KAFKA_SERVER = ["KAFKA_BOOTSTRAP_SERVER", "KAFKA_SERVER"] @@ -37,7 +37,7 @@ def context(): "include_page_view_tutorial", YN_CHOICES, ids=lambda yn: f"page_tutorial:{yn}" ) @pytest.mark.parametrize( - "faust_loglevel", FAUST_LOGLEVEL, ids=["CRITICAL", "ERROR", ] + "log_level", log_level, ids=["CRITICAL", "ERROR", ] ) @pytest.mark.parametrize("worker_port", WORKER_PORT, ids=lambda yn: f"worker_port:{yn}") @pytest.mark.parametrize( @@ -60,7 +60,7 @@ def context_combination( use_docker, include_docker_compose, include_page_view_tutorial, - faust_loglevel, + log_level, worker_port, kafka_server_environment_variable, include_codec_example, @@ -74,7 +74,7 @@ def context_combination( "use_docker": use_docker, "include_docker_compose": include_docker_compose, "include_page_view_tutorial": include_page_view_tutorial, - "faust_loglevel": faust_loglevel, + "log_level": log_level, "worker_port": worker_port, "kafka_server_environment_variable": kafka_server_environment_variable, "include_codec_example": include_codec_example, diff --git a/{{cookiecutter.project_slug}}/.gitignore b/{{cookiecutter.project_slug}}/.gitignore index d4fae38..0182963 100644 --- a/{{cookiecutter.project_slug}}/.gitignore +++ b/{{cookiecutter.project_slug}}/.gitignore @@ -73,15 +73,6 @@ ENV/ # mypy .mypy_cache/ - -### Node template -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* - # Runtime data pids *.pid @@ -94,47 +85,6 @@ lib-cov # Coverage directory used by tools like istanbul coverage -# nyc test coverage -.nyc_output - -# Bower dependency directory (https://bower.io/) -bower_components - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (http://nodejs.org/api/addons.html) -build/Release - -# Dependency directories -node_modules/ -jspm_packages/ - -# Typescript v1 declaration files -typings/ - -# Optional npm cache directory -.npm - -# Optional eslint cache -.eslintcache - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - - -### Linux template -*~ - -# temporary files which can be created if a process still has a handle open of a deleted file -.fuse_hidden* - # KDE directory preferences .directory @@ -144,7 +94,6 @@ typings/ # .nfs files are created when an open file is removed but is still being accessed .nfs* - ### VisualStudioCode template .vscode/* !.vscode/settings.json @@ -152,118 +101,8 @@ typings/ !.vscode/launch.json !.vscode/extensions.json - -{% if cookiecutter.use_pycharm == 'y' -%} -# Provided default Pycharm Run/Debug Configurations should be tracked by git -# In case of local modifications made by Pycharm, use update-index command -# for each changed file, like this: -# git update-index --assume-unchanged .idea/{{cookiecutter.project_slug}}.iml -### JetBrains template -# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm -# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 - # User-specific stuff: -.idea/**/workspace.xml -.idea/**/tasks.xml -.idea/dictionaries - -# Sensitive or high-churn files: -.idea/**/dataSources/ -.idea/**/dataSources.ids -.idea/**/dataSources.xml -.idea/**/dataSources.local.xml -.idea/**/sqlDataSources.xml -.idea/**/dynamic.xml -.idea/**/uiDesigner.xml - -# Gradle: -.idea/**/gradle.xml -.idea/**/libraries - -# CMake -cmake-build-debug/ - -# Mongo Explorer plugin: -.idea/**/mongoSettings.xml - -## File-based project format: -*.iws - -## Plugin-specific files: - -# IntelliJ -out/ - -# mpeltonen/sbt-idea plugin -.idea_modules/ - -# JIRA plugin -atlassian-ide-plugin.xml - -# Cursive Clojure plugin -.idea/replstate.xml - -# Crashlytics plugin (for Android Studio and IntelliJ) -com_crashlytics_export_strings.xml -crashlytics.properties -crashlytics-build.properties -fabric.properties -{% endif %} - - -### Windows template -# Windows thumbnail cache files -Thumbs.db -ehthumbs.db -ehthumbs_vista.db - -# Dump file -*.stackdump - -# Folder config file -Desktop.ini - -# Recycle Bin used on file shares -$RECYCLE.BIN/ - -# Windows Installer files -*.cab -*.msi -*.msm -*.msp - -# Windows shortcuts -*.lnk - - -### macOS template -# General -*.DS_Store -.AppleDouble -.LSOverride - -# Icon must end with two \r -Icon - -# Thumbnails -._* - -# Files that might appear in the root of a volume -.DocumentRevisions-V100 -.fseventsd -.Spotlight-V100 -.TemporaryItems -.Trashes -.VolumeIcon.icns -.com.apple.timemachine.donotpresent - -# Directories potentially created on remote AFP share -.AppleDB -.AppleDesktop -Network Trash Folder -Temporary Items -.apdisk - +.idea ### SublimeText template # Cache files for Sublime Text @@ -274,12 +113,6 @@ Temporary Items # Workspace files are user-specific *.sublime-workspace -# Project files should be checked into the repository, unless a significant -# proportion of contributors will probably not be using Sublime Text -# *.sublime-project - -# SFTP configuration file -sftp-config.json # Package control specific files Package Control.last-run @@ -321,7 +154,6 @@ tags [Ii]nclude [Ll]ib [Ll]ib64 -[Ss]cripts pyvenv.cfg pip-selfcheck.json .env diff --git a/{{cookiecutter.project_slug}}/.travis.yml b/{{cookiecutter.project_slug}}/.travis.yml index 960e755..d63be54 100644 --- a/{{cookiecutter.project_slug}}/.travis.yml +++ b/{{cookiecutter.project_slug}}/.travis.yml @@ -9,4 +9,4 @@ install: pip install -r requirements.txt matrix: include: - name: Test Python - script: ./scripts/test.sh + script: ./scripts/test diff --git a/{{cookiecutter.project_slug}}/README.md b/{{cookiecutter.project_slug}}/README.md index 6255268..44330c6 100644 --- a/{{cookiecutter.project_slug}}/README.md +++ b/{{cookiecutter.project_slug}}/README.md @@ -38,7 +38,7 @@ LOGGING = { }, 'handlers': { 'console': { - 'level': '{{cookiecutter.faust_loglevel}}', + 'level': '{{cookiecutter.log_level}}', 'class': 'logging.StreamHandler', 'formatter': 'default', }, @@ -46,7 +46,7 @@ LOGGING = { 'loggers': { '{{cookiecutter.project_slug}}': { 'handlers': ['console'], - 'level': '{{cookiecutter.faust_loglevel}}', + 'level': '{{cookiecutter.log_level}}', }, }, } diff --git a/{{cookiecutter.project_slug}}/k8s/.helmignore b/{{cookiecutter.project_slug}}/k8s/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/{{cookiecutter.project_slug}}/k8s/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/{{cookiecutter.project_slug}}/k8s/Chart.yaml b/{{cookiecutter.project_slug}}/k8s/Chart.yaml new file mode 100644 index 0000000..d755dfb --- /dev/null +++ b/{{cookiecutter.project_slug}}/k8s/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: "{{cookiecutter.project_slug}}" +version: "{{cookiecutter.version}}" diff --git a/{{cookiecutter.project_slug}}/k8s/templates/_helpers.tpl b/{{cookiecutter.project_slug}}/k8s/templates/_helpers.tpl new file mode 100644 index 0000000..2208f29 --- /dev/null +++ b/{{cookiecutter.project_slug}}/k8s/templates/_helpers.tpl @@ -0,0 +1,33 @@ +{{ '{{/* vim: set filetype=mustache: */}}' }} +{{ '{{/* +Expand the name of the chart. +*/}}' }} + +{{ '{{- define "k8s.name" -}}' }} +{{ '{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}' }} +{{ '{{- end -}}' }} + +{{ '{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}}' }} +{{ '{{- define "k8s.fullname" -}}' }} +{{ '{{- if .Values.fullnameOverride -}}' }} +{{ '{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}' }} +{{ '{{- else -}}' }} +{{ '{{- $name := default .Chart.Name .Values.nameOverride -}}' }} +{{ '{{- if contains $name .Release.Name -}}' }} +{{ '{{- .Release.Name | trunc 63 | trimSuffix "-" -}}' }} +{{ '{{- else -}}' }} +{{ '{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}' }} +{{ '{{- end -}}' }} +{{ '{{- end -}}' }} +{{ '{{- end -}}' }} + +{{ '{{/* +Create chart name and version as used by the chart label. +*/}}' }} +{{ '{{- define "k8s.chart" -}}' }} +{{ '{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}' }} +{{ '{{- end -}}' }} diff --git a/{{cookiecutter.project_slug}}/k8s/templates/configmap.yaml b/{{cookiecutter.project_slug}}/k8s/templates/configmap.yaml new file mode 100644 index 0000000..48c4410 --- /dev/null +++ b/{{cookiecutter.project_slug}}/k8s/templates/configmap.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ '{{ include "k8s.name" . }}' }}-configmap + labels: + app: {{ '{{ include "k8s.name" . }}' }} +data: + ENVIRONMENT: "{{ '{{ .Values.env }}' }}" + DEBUG: "{{ '{{ .Values.worker.debug }}' }}" + SSL_ENABLED: "true" + WORKER_WEB_PORT: "{{ '{{ .Values.worker.webPort }}' }}" + WORKER_LOG_LEVEL: "{{ '{{ .Values.worker.logLevel }}' }}" + KAFKA_BOOTSTRAP_SERVER: '{{ "{{ tpl (toJson .Values.kafka.bootstrapServers) . }}" }}' + KAFKA_SSL_CERT: "{{ '{{ .Values.kafka.certsPath }}/{{ .Values.kafka.certName }}' }}" + KAFKA_SSL_KEY: "{{ '{{ .Values.kafka.certsPath }}/{{ .Values.kafka.keyName }}' }}" + KAFKA_SSL_CABUNDLE: "{{ '{{ .Values.kafka.certsPath }}/{{ .Values.kafka.caName }}' }}" + SSL_ENABLED: "True" diff --git a/{{cookiecutter.project_slug}}/k8s/templates/deployment.yaml b/{{cookiecutter.project_slug}}/k8s/templates/deployment.yaml new file mode 100644 index 0000000..8b68e67 --- /dev/null +++ b/{{cookiecutter.project_slug}}/k8s/templates/deployment.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: {{ '{{ include "k8s.name" . }}' }}-deployment + labels: + app: {{ '{{ include "k8s.name" . }}' }} +spec: + replicas: {{ '{{ .Values.replicaCount }}' }} + selector: + matchLabels: + app.kubernetes.io/name: {{ '{{ include "k8s.name" . }}' }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ '{{ include "k8s.name" . }}' }} + spec: + volumes: + - name: kafka-certs + secret: + secretName: {{ '{{ .Values.kafka.certName }}' }} + containers: + - name: {{ '{{ .Chart.Name }}' }} + image: {{ '{{ .Values.image.repository }}:{{ .Values.image.tag }}' }} + imagePullPolicy: {{ '{{ .Values.image.pullPolicy }}' }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + ports: + - name: http + containerPort: {{ '{{ .Values.worker.web_port }}' }} + protocol: TCP + envFrom: + - configMapRef: + name: {{ '{{ include "k8s.name" . }}' }}-configmap + volumeMounts: + - name: kafka-certs + mountPath: {{ '{{ .Values.kafka.certsPath }}' }} + readOnly: true + command: ["./run.sh", "-c"] + resources: +{{ '{{ toYaml .Values.resources | indent 12 }}' }} + {{ '{{- with .Values.nodeSelector }}' }} + nodeSelector: +{{ '{{ toYaml . | indent 8 }}' }} + {{ '{{- end }}' }} + {{ '{{- with .Values.affinity }}' }} + affinity: +{{ '{{ toYaml . | indent 8 }}' }} + {{ '{{- end }}' }} + {{ '{{- with .Values.tolerations }}' }} + tolerations: +{{ '{{ toYaml . | indent 8 }}' }} + {{ '{{- end }}' }} diff --git a/{{cookiecutter.project_slug}}/k8s/templates/ingress.yaml b/{{cookiecutter.project_slug}}/k8s/templates/ingress.yaml new file mode 100644 index 0000000..c4608f5 --- /dev/null +++ b/{{cookiecutter.project_slug}}/k8s/templates/ingress.yaml @@ -0,0 +1,21 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ '{{ include "k8s.name" . }}' }}-ingress + labels: + app: {{ '{{ include "k8s.name" . }}' }} + annotations: + kubernetes.io/ingress.class: "nginx" +spec: + tls: + - hosts: +{%- raw %} + - {{ .Values.hostname }} + rules: + - host: {{ .Values.hostname }} + http: + paths: + - path: / + backend: + serviceName: {{ .Chart.Name }}-service +{%- endraw %} diff --git a/{{cookiecutter.project_slug}}/k8s/templates/service.yaml b/{{cookiecutter.project_slug}}/k8s/templates/service.yaml new file mode 100644 index 0000000..1b8b360 --- /dev/null +++ b/{{cookiecutter.project_slug}}/k8s/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ '{{ include "k8s.name" . }}' }}-service + labels: + app: {{ '{{ include "k8s.name" . }}' }} +spec: + type: {{ '{{ .Values.service.type }}' }} + ports: + - port: {{ '{{ .Values.service.port }}' }} + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: {{ '{{ include "k8s.name" . }}' }} diff --git a/{{cookiecutter.project_slug}}/k8s/values.yaml b/{{cookiecutter.project_slug}}/k8s/values.yaml new file mode 100644 index 0000000..43415f7 --- /dev/null +++ b/{{cookiecutter.project_slug}}/k8s/values.yaml @@ -0,0 +1,50 @@ +# Default values for faust. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: {{cookiecutter.project_slug}} + tag: 0.0.1 + pullPolicy: Always + +nameOverride: "" +fullnameOverride: "" + +service: + type: ClusterIP + port: 80 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +kafka: + sslEnabled: false + certsPath: /kafka/certs + certName: kafka-certs + caName: ca.crt + certName: certificate.pem + keyName: certificate.key + bootstrapServers: + - kafka:9092 + +worker: + webPort: {{cookiecutter.worker_port}} + logLevel: {{cookiecutter.log_level}} + debug: true diff --git a/{{cookiecutter.project_slug}}/k8s/values/prod.yaml b/{{cookiecutter.project_slug}}/k8s/values/prod.yaml new file mode 100644 index 0000000..0b95e80 --- /dev/null +++ b/{{cookiecutter.project_slug}}/k8s/values/prod.yaml @@ -0,0 +1,10 @@ +replicas: 2 + +hostname: '{{ cookiecutter.project_slug }}.com' +env: prod + +worker: + debug: false + +bootstrapServers: + - kafka:9093 diff --git a/{{cookiecutter.project_slug}}/scripts/README.md b/{{cookiecutter.project_slug}}/scripts/README.md index 09d6816..c265d74 100644 --- a/{{cookiecutter.project_slug}}/scripts/README.md +++ b/{{cookiecutter.project_slug}}/scripts/README.md @@ -1,4 +1,5 @@ # Development Scripts * `scripts/test` - Run the test suite. +* `scripts/lint` - Run the code linting. * `scripts/clean` - Remove docker containers and network. diff --git a/{{cookiecutter.project_slug}}/scripts/clean b/{{cookiecutter.project_slug}}/scripts/clean new file mode 100755 index 0000000..beff4a0 --- /dev/null +++ b/{{cookiecutter.project_slug}}/scripts/clean @@ -0,0 +1,17 @@ +#!/bin/sh -e + +if [ -d 'dist' ] ; then + rm -rf dist +fi + +if [ -d 'site' ] ; then + rm -rf site +fi + +if [ -d 'htmlcov' ] ; then + rm -rf htmlcov +fi + +# delete python cache +find . -iname '*.pyc' -delete +find . -iname '__pycache__' -delete diff --git a/{{cookiecutter.project_slug}}/scripts/lint b/{{cookiecutter.project_slug}}/scripts/lint new file mode 100755 index 0000000..cc8ec67 --- /dev/null +++ b/{{cookiecutter.project_slug}}/scripts/lint @@ -0,0 +1,9 @@ +#!/bin/sh -e + +export PREFIX="" +if [ -d 'venv' ] ; then + export PREFIX="venv/bin/" +fi + +${PREFIX}isort --multi-line=3 --trailing-comma --force-grid-wrap=0 --combine-as --line-width 120 --recursive --apply {{cookiecutter.project_slug}} tests +${PREFIX}black {{cookiecutter.project_slug}} tests diff --git a/{{cookiecutter.project_slug}}/scripts/test b/{{cookiecutter.project_slug}}/scripts/test new file mode 100755 index 0000000..0d209df --- /dev/null +++ b/{{cookiecutter.project_slug}}/scripts/test @@ -0,0 +1,10 @@ +#!/bin/sh -e + +export PREFIX="" +if [ -d 'venv' ] ; then + export PREFIX="venv/bin/" +fi + +${PREFIX}pytest ${1-"./tests"} +${PREFIX}black {{cookiecutter.project_slug}} tests/ --check +${PREFIX}flake8 {{cookiecutter.project_slug}} tests/ diff --git a/{{cookiecutter.project_slug}}/scripts/test.sh b/{{cookiecutter.project_slug}}/scripts/test.sh deleted file mode 100755 index 7e0ab62..0000000 --- a/{{cookiecutter.project_slug}}/scripts/test.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -e - -export PREFIX="" -if [ -d 'venv' ] ; then - export PREFIX="venv/bin/" -fi - -export VERSION_SCRIPT="import sys; print('%s.%s' % sys.version_info[0:2])" -export PYTHON_VERSION=`python -c "$VERSION_SCRIPT"` -export SIMPLE_SETTINGS="{{cookiecutter.project_slug}}.settings" - -set -x - -PYTHONPATH=. ${PREFIX}pytest ${1-"./tests"} diff --git a/{{cookiecutter.project_slug}}/setup.cfg b/{{cookiecutter.project_slug}}/setup.cfg index f4fde7a..7367474 100644 --- a/{{cookiecutter.project_slug}}/setup.cfg +++ b/{{cookiecutter.project_slug}}/setup.cfg @@ -1,13 +1,15 @@ [flake8] -max-line-length = 100 +max-line-length = 120 exclude = .tox,.git,docs +select = B,E,F,W,C,W504,B902,B903,B950 +ignore = E203,E231,E902,W503 [pycodestyle] -max-line-length = 100 +max-line-length = 120 exclude = .tox,.git,docs [mypy] -python_version = 3.6 +python_version = 3.7 check_untyped_defs = True ignore_errors = False ignore_missing_imports = True diff --git a/{{cookiecutter.project_slug}}/setup.py b/{{cookiecutter.project_slug}}/setup.py index eaea412..eb96a35 100644 --- a/{{cookiecutter.project_slug}}/setup.py +++ b/{{cookiecutter.project_slug}}/setup.py @@ -2,13 +2,13 @@ requires = [ {% if cookiecutter.include_rocksdb.lower() == "y" %} - "faust[rocksdb]==1.9.0",{% else %} - "faust==1.9.0",{% endif %} - "mode==4.1.3", - "robinhood-aiokafka==1.1.3", - "simple-settings==0.16.0",{% if cookiecutter.include_codec_example.lower() == "y" %} - "msgpack==0.6.1",{% endif %}{% if cookiecutter.include_schema_registry.lower() == "y" %} - "python-schema-registry-client==1.2.4",{% endif %} + "faust[rocksdb]",{% else %} + "faust",{% endif %} + "mode", + "robinhood-aiokafka", + "simple-settings",{% if cookiecutter.include_codec_example.lower() == "y" %} + "msgpack",{% endif %}{% if cookiecutter.include_schema_registry.lower() == "y" %} + "python-schema-registry-client",{% endif %} ] setup( @@ -33,7 +33,7 @@ ], {% if cookiecutter.include_codec_example.lower() == "y" %} "faust.codecs": [ - "msgpack_codec = {{cookiecutter.project_slug}}.codecs.codec:msgpack", + "msgpack_codec = {{cookiecutter.project_slug}}.codecs.codec:msgpack_codec", # add entries here to add more custom codecs ], {% endif %} diff --git a/{{cookiecutter.project_slug}}/test_requirements.txt b/{{cookiecutter.project_slug}}/test_requirements.txt index 7b00fd5..fa6a5ab 100644 --- a/{{cookiecutter.project_slug}}/test_requirements.txt +++ b/{{cookiecutter.project_slug}}/test_requirements.txt @@ -1,15 +1,15 @@ -faust==1.9.0 -simple-settings==0.16.0 +faust +simple-settings # Code quality # ------------------------------------------------------------------------------ -black==19.10b0 -flake8==3.7.9 +black +flake8 # Testing # ------------------------------------------------------------------------------ -mypy==0.761 -pytest==5.3.2 -pytest_cases==1.6.2 -pytest-cookies==0.3.0 -pytest-asyncio==0.10.0 +mypy +pytest +pytest_cases +pytest-cookies +pytest-asyncio diff --git a/{{cookiecutter.project_slug}}/tests/conftest.py b/{{cookiecutter.project_slug}}/tests/conftest.py index 6351ef8..3e88820 100644 --- a/{{cookiecutter.project_slug}}/tests/conftest.py +++ b/{{cookiecutter.project_slug}}/tests/conftest.py @@ -7,6 +7,6 @@ def test_app(event_loop): """passing in event_loop helps avoid 'attached to a different loop' error""" app.finalize() - app.conf.store = 'memory://' + app.conf.store = "memory://" app.flow_control.resume() return app diff --git a/{{cookiecutter.project_slug}}/tests/test_page_views.py b/{{cookiecutter.project_slug}}/tests/test_page_views.py index 100a7ee..47a4819 100644 --- a/{{cookiecutter.project_slug}}/tests/test_page_views.py +++ b/{{cookiecutter.project_slug}}/tests/test_page_views.py @@ -7,12 +7,12 @@ @pytest.mark.asyncio() async def test_count_page_views(test_app): async with count_page_views.test_context() as agent: - page_view = PageView(id='1', user='test') - page_view_2 = PageView(id='1', user='test2') + page_view = PageView(id="1", user="test") + page_view_2 = PageView(id="1", user="test2") await agent.put(page_view) # windowed table: we select window relative to the current event - assert page_views['1'] == 1 + assert page_views["1"] == 1 await agent.put(page_view_2) - assert page_views['1'] == 2 + assert page_views["1"] == 2 diff --git a/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/codecs/codec.py b/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/codecs/codec.py index 173a404..0cf2bdb 100644 --- a/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/codecs/codec.py +++ b/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/codecs/codec.py @@ -15,5 +15,5 @@ def _loads(self, s: bytes) -> typing.Any: return msgpack.loads(s) -def msgpack() -> codecs.Codec: +def msgpack_codec() -> codecs.Codec: return raw_msgpack() | codecs.binary() diff --git a/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/settings.py b/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/settings.py index b11a211..82ee4d6 100644 --- a/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/settings.py +++ b/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/settings.py @@ -25,16 +25,13 @@ }, "handlers": { "console": { - "level": "{{cookiecutter.faust_loglevel}}", + "level": "{{cookiecutter.log_level}}", "class": "logging.StreamHandler", "formatter": "default", } }, "loggers": { # fmt: off - "{{cookiecutter.project_slug}}": { - "handlers": ["console"], - "level": "{{cookiecutter.faust_loglevel}}", - } + "{{cookiecutter.project_slug}}": {"handlers": ["console"], "level": "{{cookiecutter.log_level}}",} }, } @@ -46,18 +43,17 @@ if SSL_ENABLED: # file in pem format containing the client certificate, as well as any ca certificates # needed to establish the certificate’s authenticity - SSL_CERT_FILE = "path_to_ssl_certificate.pem" + KAFKA_SSL_CERT = os.getenv("KAFKA_SSL_CERT", "path_to_ssl_certificate.pem") # filename containing the client private key - SSL_KEY_FILE = "path_tp_private_key.key" + KAFKA_SSL_KEY = os.getenv("KAFKA_SSL_KEY", "path_tp_private_key.key") # filename of ca file to use in certificate verification - CA_FILE = "path_to_ca_file.crt" + KAFKA_SSL_CABUNDLE = os.getenv("KAFKA_SSL_CABUNDLE", "path_to_ca_file.crt") # password for decrypting the client private key SSL_KEY_PASSWORD = os.getenv("SSL_KEY_PASSWORD") SSL_CONTEXT = ssl.create_default_context( - purpose=ssl.Purpose.SERVER_AUTH, cafile=CA_FILE) - SSL_CONTEXT.load_cert_chain(SSL_CERT_FILE, keyfile=SSL_KEY_FILE, password=SSL_KEY_PASSWORD) -{% endif %} + purpose=ssl.Purpose.SERVER_AUTH, cafile=KAFKA_SSL_CABUNDLE) + SSL_CONTEXT.load_cert_chain(KAFKA_SSL_CERT, keyfile=KAFKA_SSL_KEY, password=SSL_KEY_PASSWORD){% endif %} \ No newline at end of file