From 02d3449acf14648377b74e1cff8a13c210999c5f Mon Sep 17 00:00:00 2001 From: Austin Pray Date: Tue, 12 Dec 2017 11:45:24 -0600 Subject: [PATCH] Converts kizuna to worker based model Use slack events API to have events pushed to kizuna rather than pulled from rtm api api process serves a lightweight json api that enqueues messages on rabbitmq worker process uses dramatiq actors to async take care of messages --- Dockerfile.api | 5 + Dockerfile.base | 10 +- Dockerfile.web | 2 +- Dockerfile.bot => Dockerfile.worker | 2 +- Makefile | 98 +++++++++++++----- api.py | 55 ++++++++++ bot.py | 128 ------------------------ config/__init__.py | 3 +- config/gunicorn_api.py | 7 ++ config/{gunicorn.py => gunicorn_web.py} | 0 docker-compose.yml | 19 +++- kizuna_web/app.py | 2 +- requirements.txt | 77 ++++---------- worker.py | 90 +++++++++++++++++ 14 files changed, 275 insertions(+), 223 deletions(-) create mode 100644 Dockerfile.api rename Dockerfile.bot => Dockerfile.worker (59%) create mode 100644 api.py delete mode 100644 bot.py create mode 100644 config/gunicorn_api.py rename config/{gunicorn.py => gunicorn_web.py} (100%) create mode 100644 worker.py diff --git a/Dockerfile.api b/Dockerfile.api new file mode 100644 index 00000000..33ca5823 --- /dev/null +++ b/Dockerfile.api @@ -0,0 +1,5 @@ +FROM austinpray/kizuna/base + +CMD ["gunicorn", "--config", "python:config.gunicorn_api", "api:app"] + +COPY . ${workdir} diff --git a/Dockerfile.base b/Dockerfile.base index c3ff7fcf..80291b25 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -8,8 +8,10 @@ ENV PYTHONPATH="${PYTHONPATH}:${workdir}" RUN apt-get update \ && apt-get install -y vim graphviz -COPY requirements.txt ${workdir}/requirements.txt - -RUN pip --no-cache-dir install -r requirements.txt - +# heavy deps +RUN pip install spacy RUN python -m spacy download en + +# regular deps +COPY requirements.txt ${workdir}/requirements.txt +RUN pip install -r requirements.txt diff --git a/Dockerfile.web b/Dockerfile.web index 210893a2..350ccbbb 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -1,5 +1,5 @@ FROM austinpray/kizuna/base -CMD ["gunicorn", "--config", "python:config.gunicorn", "web:app"] +CMD ["gunicorn", "--config", "python:config.gunicorn_web", "web:app"] COPY . ${workdir} diff --git a/Dockerfile.bot b/Dockerfile.worker similarity index 59% rename from Dockerfile.bot rename to Dockerfile.worker index c95e1ff7..072a14ab 100644 --- a/Dockerfile.bot +++ b/Dockerfile.worker @@ -1,5 +1,5 @@ FROM austinpray/kizuna/base -CMD ["python", "-u", "./bot.py"] +CMD ["dramatiq", "worker"] COPY . ${workdir} diff --git a/Makefile b/Makefile index b0886404..02f9b9b5 100644 --- a/Makefile +++ b/Makefile @@ -1,51 +1,87 @@ -.PHONY: kub_deploy pep8 registry_push web bot build pull perm dev_info base migrate_dev +.PHONY: kub_deploy pep8 registry_push web api build pull perm dev_info base migrate_dev +# simulate CI environment +TRAVIS_COMMIT ?= $(shell git rev-parse HEAD) + +# tags used locally base_tag = austinpray/kizuna/base -bot_tag = austinpray/kizuna/bot + +api_tag = austinpray/kizuna/api web_tag = austinpray/kizuna/web +worker_tag = austinpray/kizuna/worker -registry_base_url = registry.heroku.com/kizunaai -registry_base_url = us.gcr.io/kizuna-188702 +# remote registry prefix for pushing to gcloud +registry_prefix = us.gcr.io/kizuna-188702 -TRAVIS_COMMIT ?= $(shell git rev-parse HEAD) +# base tags for different images in project +registry_base_tag = $(registry_prefix)/base -registry_base_tag = $(registry_base_url)/base -registry_bot_tag = $(registry_base_url)/bot -registry_web_tag = $(registry_base_url)/web +registry_api_tag = $(registry_prefix)/api +registry_web_tag = $(registry_prefix)/web +registry_worker_tag = $(registry_prefix)/worker +# commit level tag registry_base_tag_commit = $(registry_base_tag):$(TRAVIS_COMMIT) -registry_bot_tag_commit = $(registry_bot_tag):$(TRAVIS_COMMIT) + +registry_api_tag_commit = $(registry_api_tag):$(TRAVIS_COMMIT) registry_web_tag_commit = $(registry_web_tag):$(TRAVIS_COMMIT) +registry_worker_tag_commit = $(registry_worker_tag):$(TRAVIS_COMMIT) +# latest tags registry_base_tag_latest = $(registry_base_tag):latest -registry_bot_tag_latest = $(registry_bot_tag):latest + +registry_api_tag_latest = $(registry_api_tag):latest registry_web_tag_latest = $(registry_web_tag):latest +registry_worker_tag_latest = $(registry_worker_tag):latest -build: dev_info bot web +# build everything +build: dev_info api worker web +# check the project for pep8 compliance pep8: docker run --rm -v $(shell pwd):/code omercnet/pycodestyle --show-source /code +# make a dev-info file so kizuna knows what commit she's on +dev_info: + bin/generate-dev-info.py --revision $(TRAVIS_COMMIT) > .dev-info.json + +# push all the images to gcloud registry registry_push: docker tag $(base_tag) $(registry_base_tag_commit) - docker tag $(bot_tag) $(registry_bot_tag_commit) + docker tag $(api_tag) $(registry_api_tag_commit) docker tag $(web_tag) $(registry_web_tag_commit) + docker tag $(worker_tag) $(registry_worker_tag_commit) gcloud docker -- push $(registry_base_tag_commit) - gcloud docker -- push $(registry_bot_tag_commit) + gcloud docker -- push $(registry_api_tag_commit) gcloud docker -- push $(registry_web_tag_commit) - + gcloud docker -- push $(registry_worker_tag_commit) + docker tag $(base_tag) $(registry_base_tag_latest) + docker tag $(api_tag) $(registry_api_tag_latest) + docker tag $(web_tag) $(registry_web_tag_latest) + docker tag $(worker_tag) $(registry_worker_tag_latest) + gcloud docker -- push $(registry_base_tag_latest) + gcloud docker -- push $(registry_api_tag_latest) + gcloud docker -- push $(registry_web_tag_latest) + gcloud docker -- push $(registry_worker_tag_latest) + +# release the current commit to kube kube_deploy: registry_push - kubectl set image deployment/bot bot=$(registry_bot_tag_commit) + kubectl set image deployment/api api=$(registry_api_tag_commit) kubectl set image deployment/web web=$(registry_web_tag_commit) + kubectl set image deployment/worker worker=$(registry_worker_tag_commit) +# pull images from registry prolly for caching reasons pull: gcloud docker -- pull $(registry_base_tag) - gcloud docker -- pull $(registry_bot_tag) + gcloud docker -- pull $(registry_api_tag) gcloud docker -- pull $(registry_web_tag) + gcloud docker -- pull $(registry_worker_tag) docker tag $(registry_base_tag) $(base_tag) - docker tag $(registry_bot_tag) $(bot_tag) + docker tag $(registry_api_tag) $(api_tag) docker tag $(registry_web_tag) $(web_tag) + docker tag $(registry_worker_tag) $(worker_tag) +# image building base: docker build \ --file Dockerfile.base \ @@ -53,14 +89,14 @@ base: -t $(base_tag) \ . -bot: base +api: base docker build \ - --file Dockerfile.bot \ - --cache-from $(bot_tag) \ - -t $(bot_tag) \ + --file Dockerfile.api \ + --cache-from $(api_tag) \ + -t $(api_tag) \ . -web: bot +web: api docker run -it --rm --name build-web-assets -v $(shell pwd):/kizuna -w /kizuna node:9 npm run build docker build \ --file Dockerfile.web \ @@ -68,14 +104,22 @@ web: bot -t $(web_tag) \ . +worker: base + docker build \ + --file Dockerfile.worker \ + --cache-from $(worker_tag) \ + -t $(worker_tag) \ + . + +# docker permissions helper perm: sudo chown -R $(shell whoami):$(shell whoami) . -dev_info: - bin/generate-dev-info.py --revision $(TRAVIS_COMMIT) > .dev-info.json - +# dev commands +## watch for file changes and restart accordingly dev: - nodemon -e 'py' --exec docker-compose restart bot web + nodemon -e 'py' --exec docker-compose restart api web +## run dev migrations migrate_dev: - docker-compose run bot alembic upgrade head + docker-compose run api alembic upgrade head diff --git a/api.py b/api.py new file mode 100644 index 00000000..67d11422 --- /dev/null +++ b/api.py @@ -0,0 +1,55 @@ +import falcon +import json +import config +import logging + +from worker import worker + + +class HealthCheckResource(object): + def on_get(self, req, resp): + resp.body = '{"ok": true}' + + +class EventsResource(object): + + def __init__(self): + self.logger = logging.getLogger('thingsapp.' + __name__) + + def on_post(self, req, resp): + if not req.content_length: + resp.status = falcon.HTTP_400 + resp.body = 'go away' + return + + doc = json.load(req.stream) + print(doc) + + callback_type = doc['type'] + + if doc['token'] != config.SLACK_VERIFICATION_TOKEN: + resp.status = falcon.HTTP_401 + resp.body = 'go away' + return + + if callback_type == 'url_verification': + resp.body = doc['challenge'] + return + + if callback_type == 'event_callback': + event = doc['event'] + event_type = event['type'] + if event_type == 'message': + self.logger.debug(event) + worker.send(event) + + resp.body = 'thanks!' + + +app = falcon.API() + +events = EventsResource() +app.add_route('/slack/events', events) + +healthChecks = HealthCheckResource() +app.add_route('/', healthChecks) diff --git a/bot.py b/bot.py deleted file mode 100644 index 7ecc41fa..00000000 --- a/bot.py +++ /dev/null @@ -1,128 +0,0 @@ -import os -import signal -import sys -import time -import traceback -import backoff -from sqlalchemy.exc import OperationalError as DatabaseOperationalError - -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from slackclient import SlackClient -from slackclient.server import SlackConnectionError -from requests import ConnectionError - -from kizuna.Kizuna import Kizuna -from kizuna.commands.AtGraphCommand import AtGraphCommand -from kizuna.commands.AtGraphDataCollector import AtGraphDataCollector -from kizuna.commands.ClapCommand import ClapCommand -from kizuna.commands.LoginCommand import LoginCommand -from kizuna.commands.PingCommand import PingCommand -from kizuna.commands.ReactCommand import ReactCommand -from kizuna.commands.UserRefreshCommand import UserRefreshCommand -from kizuna.strings import HAI_DOMO, GOODBYE - -import spacy - -from raven import Client -import config - - -def signal_handler(signal, frame): - print("\n{}".format(GOODBYE)) - sys.exit(0) - - -READ_WEBSOCKET_DELAY = 0.01 - -nlp = spacy.load('en') - -if __name__ == "__main__": - signal.signal(signal.SIGINT, signal_handler) - signal.signal(signal.SIGTERM, signal_handler) - - DEV_INFO = Kizuna.read_dev_info('./.dev-info.json') - - sentry = Client(config.SENTRY_URL, - release=DEV_INFO.get('revision'), - environment=config.KIZUNA_ENV) if config.SENTRY_URL else None - - if not config.SLACK_API_TOKEN: - raise ValueError('You are missing a slack token! Please set the SLACK_API_TOKEN environment variable in your ' - '.env file or in the system environment') - - main_loop_exceptions = (DatabaseOperationalError, - ConnectionError, - SlackConnectionError) - - def on_backoff(details): - print("Ran into trouble in the main_loop monkaS. " - "Backing off {wait:0.1f} seconds after {tries} tries ".format(**details)) - - if sentry: - sentry.captureException() - else: - print(traceback.format_exc()) - - @backoff.on_exception(backoff.expo, - main_loop_exceptions, - max_tries=8, - on_backoff=on_backoff) - def main_loop(): - sc = SlackClient(config.SLACK_API_TOKEN) - - db_engine = create_engine(config.DATABASE_URL) - Session = sessionmaker(bind=db_engine) - - if sc.rtm_connect(): - auth = sc.api_call('auth.test') - bot_id = auth['user_id'] - - k = Kizuna(bot_id, - slack_client=sc, - main_channel=config.MAIN_CHANNEL, - home_channel=config.KIZUNA_HOME_CHANNEL) - - k.handle_startup(DEV_INFO, Session()) - - pc = PingCommand() - k.register_command(pc) - - lc = LoginCommand(Session) - k.register_command(lc) - - clap = ClapCommand() - k.register_command(clap) - - at_graph_command = AtGraphCommand(Session) - k.register_command(at_graph_command) - - at_graph_data_collector = AtGraphDataCollector(Session, sc) - k.register_command(at_graph_data_collector) - - user_refresh_command = UserRefreshCommand(db_session=Session) - k.register_command(user_refresh_command) - - react_command = ReactCommand(Session, nlp=nlp) - k.register_command(react_command) - - print("{} BOT_ID {}".format(HAI_DOMO, bot_id)) - - while True: - read = sc.rtm_read() - try: - if read: - for output in read: - if output['type'] == 'message': - k.handle_message(output) - except Exception: - if sentry: - sentry.captureException() - else: - print(traceback.format_exc()) - - time.sleep(READ_WEBSOCKET_DELAY) - else: - print("Can't connect to slack.") - - main_loop() diff --git a/config/__init__.py b/config/__init__.py index 70057768..19f13260 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -14,9 +14,10 @@ SECRET_KEY = os.environ.get('SECRET_KEY', None) SENTRY_URL = os.environ.get('SENTRY_URL', None) SLACK_API_TOKEN = os.environ.get('SLACK_API_TOKEN', None) +SLACK_VERIFICATION_TOKEN = os.environ.get('SLACK_VERIFICATION_TOKEN', None) FERNET_KEY = os.environ.get('FERNET_KEY', None) if FERNET_KEY: FERNET_KEY = FERNET_KEY.encode('ascii') -FERNET_TTL = DAY_IN_SECONDS if KIZUNA_ENV != 'development' else 30*DAY_IN_SECONDS +FERNET_TTL = DAY_IN_SECONDS if KIZUNA_ENV != 'development' else 30 * DAY_IN_SECONDS diff --git a/config/gunicorn_api.py b/config/gunicorn_api.py new file mode 100644 index 00000000..53a7abd6 --- /dev/null +++ b/config/gunicorn_api.py @@ -0,0 +1,7 @@ +from . import KIZUNA_ENV + +if KIZUNA_ENV == 'development': + bind = '0.0.0.0:8001' + loglevel = 'debug' + +workers = 4 diff --git a/config/gunicorn.py b/config/gunicorn_web.py similarity index 100% rename from config/gunicorn.py rename to config/gunicorn_web.py diff --git a/docker-compose.yml b/docker-compose.yml index 186f8b05..23605316 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,15 +10,30 @@ services: POSTGRES_DB: kizuna POSTGRES_PASSWORD: kizuna POSTGRES_USER: kizuna - bot: + rabbitmq: + image: rabbitmq + worker: build: context: . - dockerfile: Dockerfile.bot + dockerfile: Dockerfile.worker env_file: .env volumes: - .:/kizuna depends_on: - db + - rabbitmq + api: + build: + context: . + dockerfile: Dockerfile.api + env_file: .env + volumes: + - .:/kizuna + depends_on: + - db + - rabbitmq + ports: + - 8001:8001 web: build: context: . diff --git a/kizuna_web/app.py b/kizuna_web/app.py index 61c3705f..19a723b2 100644 --- a/kizuna_web/app.py +++ b/kizuna_web/app.py @@ -4,7 +4,7 @@ import config from kizuna.Kizuna import Kizuna -from .views import blueprint as views_blueprint +from kizuna_web.views import blueprint as views_blueprint DEV_INFO = Kizuna.read_dev_info('./.dev-info.json') diff --git a/requirements.txt b/requirements.txt index b8270b59..5306160f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,58 +1,19 @@ -alembic==0.9.6 -arrow==0.12.0 -asn1crypto==0.23.0 -backoff==1.4.3 -blinker==1.4 -boto3==1.4.8 -botocore==1.8.7 -certifi==2017.11.5 -cffi==1.11.2 -chardet==3.0.4 -click==6.7 -cryptography==2.1.4 -cymem==1.31.2 -cytoolz==0.8.2 -dill==0.2.7.1 -docutils==0.14 -Flask==0.12.2 -ftfy==4.4.3 -gunicorn==19.7.1 -html5lib==1.0.1 -idna==2.6 -itsdangerous==0.24 -Jinja2==2.10 -jmespath==0.9.3 -Mako==1.0.7 -MarkupSafe==1.0 -msgpack-numpy==0.4.1 -msgpack-python==0.4.8 -murmurhash==0.28.0 -numpy==1.13.3 -palettable==3.1.0 -pathlib==1.0.1 -plac==0.9.6 -preshed==1.0.0 -psycopg2==2.7.3.2 -pycparser==2.18 -pygraphviz==1.3.1 -python-dateutil==2.6.1 -python-editor==1.0.3 -raven==6.3.0 -regex==2017.4.5 -requests==2.18.4 -s3transfer==0.1.12 -six==1.11.0 -slackclient==1.1.0 -spacy==2.0.5 -SQLAlchemy==1.1.15 -termcolor==1.1.0 -thinc==6.10.2 -toolz==0.8.2 -tqdm==4.19.4 -ujson==1.35 -urllib3==1.22 -wcwidth==0.1.7 -webencodings==0.5.1 -websocket-client==0.44.0 -Werkzeug==0.12.2 -wrapt==1.10.11 +Flask +SQLAlchemy +alembic +arrow +backoff +boto3 +cryptography +gunicorn +palettable +psycopg2 +pygraphviz +raven[flask] +slackclient + +dramatiq[rabbitmq, watch] + +# falcon related +falcon +ujson diff --git a/worker.py b/worker.py new file mode 100644 index 00000000..b7370a82 --- /dev/null +++ b/worker.py @@ -0,0 +1,90 @@ +import os +import signal +import sys +import time +import traceback +import backoff +from sqlalchemy.exc import OperationalError as DatabaseOperationalError + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from slackclient import SlackClient +from slackclient.server import SlackConnectionError +from requests import ConnectionError + +from kizuna.Kizuna import Kizuna +from kizuna.commands.AtGraphCommand import AtGraphCommand +from kizuna.commands.AtGraphDataCollector import AtGraphDataCollector +from kizuna.commands.ClapCommand import ClapCommand +from kizuna.commands.LoginCommand import LoginCommand +from kizuna.commands.PingCommand import PingCommand +from kizuna.commands.ReactCommand import ReactCommand +from kizuna.commands.UserRefreshCommand import UserRefreshCommand +from kizuna.strings import HAI_DOMO, GOODBYE + +import spacy + +from raven import Client +import config +import dramatiq + +from dramatiq.brokers.rabbitmq import RabbitmqBroker + +rabbitmq_broker = RabbitmqBroker(host="rabbitmq") +dramatiq.set_broker(rabbitmq_broker) + + +READ_WEBSOCKET_DELAY = 0.01 + +nlp = spacy.load('en') + +DEV_INFO = Kizuna.read_dev_info('./.dev-info.json') + +sentry = Client(config.SENTRY_URL, + release=DEV_INFO.get('revision'), + environment=config.KIZUNA_ENV) if config.SENTRY_URL else None + +if not config.SLACK_API_TOKEN: + raise ValueError('You are missing a slack token! Please set the SLACK_API_TOKEN environment variable in your ' + '.env file or in the system environment') + +sc = SlackClient(config.SLACK_API_TOKEN) +db_engine = create_engine(config.DATABASE_URL) +make_session = sessionmaker(bind=db_engine) + +auth = sc.api_call('auth.test') +bot_id = auth['user_id'] + +k = Kizuna(bot_id, + slack_client=sc, + main_channel=config.MAIN_CHANNEL, + home_channel=config.KIZUNA_HOME_CHANNEL) + +# k.handle_startup(DEV_INFO, make_session()) + +pc = PingCommand() +k.register_command(pc) + +lc = LoginCommand(make_session) +k.register_command(lc) + +clap = ClapCommand() +k.register_command(clap) + +at_graph_command = AtGraphCommand(make_session) +k.register_command(at_graph_command) + +at_graph_data_collector = AtGraphDataCollector(make_session, sc) +k.register_command(at_graph_data_collector) + +user_refresh_command = UserRefreshCommand(db_session=make_session) +k.register_command(user_refresh_command) + +react_command = ReactCommand(make_session, nlp=nlp) +k.register_command(react_command) + + +@dramatiq.actor +def worker(payload): + if payload['type'] == 'message': + k.handle_message(payload)