Skip to content

Commit

Permalink
fix: settings fixed. kafka boostrap server set as convention (#50)
Browse files Browse the repository at this point in the history
  • Loading branch information
marcosschroh authored Apr 3, 2020
1 parent 6dee655 commit 3399eb3
Show file tree
Hide file tree
Showing 10 changed files with 47 additions and 48 deletions.
3 changes: 1 addition & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ Choose from 1, 2, 3, 4, 5 (1, 2, 3, 4, 5) [1]:
use_docker [n]: y
include_docker_compose [n]: y
worker_port [6066]:
kafka_server_environment_variable [KAFKA_BOOTSTRAP_SERVER]:
Select log_level:
1 - CRITICAL
2 - ERROR
Expand All @@ -88,7 +87,7 @@ Enter the project and take a look around:
cd super_faust/
ls

CONTRIBUTORS.txt Dockerfile LICENSE Makefile README.md docker-compose.yaml run.sh setup.cfg setup.py super_faust wait_for_services.sh
CONTRIBUTORS.txt Dockerfile LICENSE Makefile README.md docker-compose.yaml setup.cfg pyproject.py super_faust
```

Now time to run it. In a terminal located at the project root directory folder execute:
Expand Down
1 change: 0 additions & 1 deletion cookiecutter.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
"use_docker": "y",
"include_docker_compose": "y",
"worker_port": 6066,
"kafka_server_environment_variable": "KAFKA_BOOTSTRAP_SERVER",
"log_level": [
"INFO",
"CRITICAL",
Expand Down
2 changes: 1 addition & 1 deletion scripts/test_docker
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,4 @@ docker network rm ${project_name} || true
./scripts/test

# Listing the agents to prove that the project generated is working
docker-compose run -e SIMPLE_SETTINGS=${project_name}.settings -e PYTHONPATH=${project_name} ${project_name} venv/bin/faust -A app agents
docker-compose run --entrypoint scripts/wait-for-services -e PYTHONPATH=${project_name} -e SIMPLE_SETTINGS=settings ${project_name} venv/bin/faust -A app agents
7 changes: 0 additions & 7 deletions tests/test_cookiecutter_generation.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,6 @@ def context():
"log_level", log_level, ids=["CRITICAL", "ERROR", ]
)
@pytest.mark.parametrize("worker_port", WORKER_PORT, ids=lambda yn: f"worker_port:{yn}")
@pytest.mark.parametrize(
"kafka_server_environment_variable",
KAFKA_SERVER,
ids=lambda yn: f"kafka_Server:{yn}",
)
@pytest.mark.parametrize(
"include_schema_registry", YN_CHOICES, ids=lambda yn: f"schema_registry:{yn}"
)
Expand All @@ -53,7 +48,6 @@ def context_combination(
include_docker_compose,
log_level,
worker_port,
kafka_server_environment_variable,
include_schema_registry,
include_rocksdb,
ci_provider,
Expand All @@ -64,7 +58,6 @@ def context_combination(
"include_docker_compose": include_docker_compose,
"log_level": log_level,
"worker_port": worker_port,
"kafka_server_environment_variable": kafka_server_environment_variable,
"include_schema_registry": include_schema_registry,
"include_rocksdb": include_rocksdb,
"ci_provider": ci_provider,
Expand Down
15 changes: 11 additions & 4 deletions {{cookiecutter.project_slug}}/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,17 @@ Settings

Settings are created based on [local-settings](https://github.com/drgarcia1986/simple-settings) package.

The only settings required if the `{{cookiecutter.kafka_server_environment_variable}}` environment variable.
The only settings required if the `KAFKA_BOOTSTRAP_SERVER` environment variable.

```python
SIMPLE_SETTINGS = {
'OVERRIDE_BY_ENV': True,
'CONFIGURE_LOGGING': True,
'REQUIRED_SETTINGS': ('{{cookiecutter.kafka_server_environment_variable}}',),
'REQUIRED_SETTINGS': ('KAFKA_BOOTSTRAP_SERVER',),
}

# The following variables can be ovirriden from ENV
{{cookiecutter.kafka_server_environment_variable}} = "kafka://kafka:9092"
KAFKA_BOOTSTRAP_SERVER = "kafka://kafka:9092"
```

The settings also include a basic logging configuration:
Expand Down Expand Up @@ -90,7 +90,7 @@ Useful ENVIRONMENT variables that you may change:
|Variable| description | example |
|--------|--------------|---------|
| WORKER_PORT | Worker port | `6066` |
| {{cookiecutter.kafka_server_environment_variable}} | Kafka servers | `kafka://kafka:9092` |
| KAFKA_BOOTSTRAP_SERVER | Kafka servers | `kafka://kafka:9092` |
| KAFKA_BOOSTRAP_SERVER_NAME | Kafka server name| `kafka` |
| KAFKA_BOOSTRAP_SERVER_PORT | Kafka server port | `9092` |
| SCHEMA_REGISTRY_SERVER | Schema registry server name | `schema-registry` |
Expand All @@ -106,6 +106,13 @@ Run tests
./scripts/test.sh
```

Lint code
---------

```sh
./scripts/lint
```

Type checks
-----------

Expand Down
2 changes: 1 addition & 1 deletion {{cookiecutter.project_slug}}/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ services:
- schema-registry {% endif %}
environment:
- WORKER_PORT={{cookiecutter.worker_port}}
- {{cookiecutter.kafka_server_environment_variable}}=kafka://kafka:9092
- KAFKA_BOOTSTRAP_SERVER=kafka://kafka:9092
- KAFKA_BOOSTRAP_SERVER_NAME=kafka
- KAFKA_BOOSTRAP_SERVER_PORT=9092 {% if cookiecutter.include_schema_registry.lower() == "y" %}
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
Expand Down
1 change: 0 additions & 1 deletion {{cookiecutter.project_slug}}/k8s/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,3 @@ data:
KAFKA_SSL_CERT: "{{ '{{ .Values.kafka.certsPath }}/{{ .Values.kafka.certName }}' }}"
KAFKA_SSL_KEY: "{{ '{{ .Values.kafka.certsPath }}/{{ .Values.kafka.keyName }}' }}"
KAFKA_SSL_CABUNDLE: "{{ '{{ .Values.kafka.certsPath }}/{{ .Values.kafka.caName }}' }}"
SSL_ENABLED: "True"
1 change: 1 addition & 0 deletions {{cookiecutter.project_slug}}/scripts/test
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ if [ -d 'venv' ] ; then
fi

export PYTHONPATH={{cookiecutter.project_slug}}
export SIMPLE_SETTINGS=settings

${PREFIX}pytest ${1-"./tests"}
${PREFIX}black {{cookiecutter.project_slug}} tests/ --check
Expand Down
13 changes: 11 additions & 2 deletions {{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/app.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
import faust

import settings
from simple_settings import settings

app = faust.App(**settings.FAUST_CONF)
app = faust.App(
id=1,
autodiscover=["page_views"],
broker=settings.KAFKA_BOOTSTRAP_SERVER,
store=settings.STORE_URI,
logging_config=settings.LOGGING,
topic_allow_declare=settings.TOPIC_ALLOW_DECLARE,
topic_disable_leader=settings.TOPIC_DISABLE_LEADER,
broker_credentials=settings.SSL_CONTEXT,
)
Original file line number Diff line number Diff line change
@@ -1,60 +1,52 @@
import os
import ssl
from logging.config import dictConfig

SIMPLE_SETTINGS = {
"OVERRIDE_BY_ENV": True,
"CONFIGURE_LOGGING": True,
"REQUIRED_SETTINGS": ("{{cookiecutter.kafka_server_environment_variable}}",),
"REQUIRED_SETTINGS": ("KAFKA_BOOTSTRAP_SERVER",),
}

# The following variables can be ovirriden from ENV
{{cookiecutter.kafka_server_environment_variable}} = "kafka://kafka:9092"
KAFKA_BOOTSTRAP_SERVER = "kafka://kafka:9092"
# SCHEMA_REGISTRY_URL = "http://schema-registry:8081"
{% if cookiecutter.include_rocksdb.lower() == "y" %}
STORE_URI = "rocksdb://"
{% else %}
STORE_URI = "memory://"
{% endif %}

LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {"default": {"format": "%(asctime)s %(levelname)s %(name)s %(message)s"}},
"handlers": {"console": {"level": "{{cookiecutter.log_level}}", "class": "logging.StreamHandler", "formatter": "default",}},
"loggers": {"page_views": {"handlers": ["console"], "level": "{{cookiecutter.log_level}}"}},
}
LOGGING = dictConfig(
{
"version": 1,
"disable_existing_loggers": False,
"formatters": {"default": {"format": "%(asctime)s %(levelname)s %(name)s %(message)s"}},
"handlers": {"console": {"level": "{{cookiecutter.log_level}}", "class": "logging.StreamHandler", "formatter": "default",}},
"loggers": {"page_views": {"handlers": ["console"], "level": "{{cookiecutter.log_level}}"}},
}
)

TOPIC_ALLOW_DECLARE = os.getenv("TOPIC_ALLOW_DECLARE", True)
TOPIC_DISABLE_LEADER = os.getenv("TOPIC_DISABLE_LEADER", False)

FAUST_CONF = {
"id": 1,
"autodiscover": ["page_views"],
"broker": {{cookiecutter.kafka_server_environment_variable}},
"store": STORE_URI,
"logging_config": dictConfig(LOGGING),
"topic_allow_declare": TOPIC_ALLOW_DECLARE,
"topic_disable_leader": TOPIC_DISABLE_LEADER,
}
TOPIC_ALLOW_DECLARE = True
TOPIC_DISABLE_LEADER = False

SSL_ENABLED = os.getenv("SSL_ENABLED", False)

SSL_ENABLED = False
SSL_CONTEXT = None

if SSL_ENABLED:
# file in pem format containing the client certificate, as well as any ca certificates
# needed to establish the certificate’s authenticity
KAFKA_SSL_CERT = os.getenv("KAFKA_SSL_CERT", "path_to_ssl_certificate.pem")
KAFKA_SSL_CERT = None

# filename containing the client private key
KAFKA_SSL_KEY = os.getenv("KAFKA_SSL_KEY", "path_tp_private_key.key")
KAFKA_SSL_KEY = None

# filename of ca file to use in certificate verification
KAFKA_SSL_CABUNDLE = os.getenv("KAFKA_SSL_CABUNDLE", "path_to_ca_file.crt")
KAFKA_SSL_CABUNDLE = None

# password for decrypting the client private key
SSL_KEY_PASSWORD = os.getenv("SSL_sKEY_PASSWORD")
SSL_KEY_PASSWORD = None

SSL_CONTEXT = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH, cafile=KAFKA_SSL_CABUNDLE)

SSL_CONTEXT.load_cert_chain(KAFKA_SSL_CERT, keyfile=KAFKA_SSL_KEY, password=SSL_KEY_PASSWORD)

FAUST_CONF.update({"broker_credentials": SSL_CONTEXT})

0 comments on commit 3399eb3

Please sign in to comment.