Meet Will.
-Will is the friendliest, easiest-to-teach bot you've ever used. He works on hipchat, in rooms and 1-1 chats.
+Will is the friendliest, easiest-to-teach bot you've ever used. He works on Slack, Hipchat, Rocket.chat, and more - so you can build your bot without platform lock-in.
He makes teaching your chat bot this simple:
@@ -14,9 +14,11 @@ def say_hello(self, message):
self.say("oh, hello!")
```
-Will was first built by [Steven Skoczen](http://stevenskoczen.com) while in the [Greenkahuna Skunkworks](http://skunkworks.greenkahuna.com), and has been [contributed to by lots of folks](http://skoczen.github.io/will/improve/#the-shoulders-of-giants).
+The first version of Will was built by [Steven Skoczen](http://stevenskoczen.com) while in the Greenkahuna Skunkworks (now defunct), was extended by [Ink and Feet](https://inkandfeet.com) and has been [contributed to by lots of awesome people](http://skoczen.github.io/will/improve/#the-shoulders-of-giants).
Will has docs, including a quickstart and lots of screenshots at:
**[http://skoczen.github.io/will/](http://skoczen.github.io/will)**
+If you've been using Will 0.x or 1.x, and are wondering how to upgrade to 2.x, [here's our friendly guide](http://skoczen.github.io/will/upgrading_to_2/). :)
+
Check them out!
diff --git a/circle.yml b/circle.yml
index 8fdde781..3d109cd3 100644
--- a/circle.yml
+++ b/circle.yml
@@ -1,8 +1,10 @@
dependencies:
pre:
- - pip install -r requirements.dev.txt
+ - pip install -r requirements.txt; pip install -r requirements/dev.txt
test:
override:
- /home/ubuntu/will/will/scripts/test_run.sh
+ - rm -rf .eggs
- coverage run -m nose
+ - tox
- flake8
\ No newline at end of file
diff --git a/config.py b/config.py
deleted file mode 100644
index 65f2c04e..00000000
--- a/config.py
+++ /dev/null
@@ -1,144 +0,0 @@
-# Welcome to Will's settings.
-#
-
-# Config and the environment:
-# ---------------------------
-# Will can use settings from the environment or this file, and sets reasonable defaults.
-#
-# Best practices: set keys and the like in the environment, and anything you'd be ok
-# with other people knowing in this file.
-#
-# To specify in the environment, just prefix with WILL_
-# (i.e. WILL_DEFAULT_ROOM becomes DEFAULT_ROOM).
-# In case of conflict, you will see a warning message, and the value in this file will win.
-
-# ------------------------------------------------------------------------------------
-# Required settings
-# ------------------------------------------------------------------------------------
-
-# The list of plugin modules will should load.
-# Will recursively loads all plugins contained in each module.
-
-
-# This list can contain:
-#
-# Built-in core plugins:
-# ----------------------
-# All built-in modules: will.plugins
-# Built-in modules: will.plugins.module_name
-# Specific plugins: will.plugins.module_name.plugin
-#
-# Plugins in your will:
-# ----------------------
-# All modules: plugins
-# A specific module: plugins.module_name
-# Specific plugins: plugins.module_name.plugin
-#
-# Plugins anywhere else on your PYTHONPATH:
-# -----------------------------------------
-# All modules: someapp
-# A specific module: someapp.module_name
-# Specific plugins: someapp.module_name.plugin
-
-
-# By default, the list below includes all the core will plugins and
-# all your project's plugins.
-
-PLUGINS = [
- # Built-ins
- "will.plugins.admin",
- "will.plugins.chat_room",
- "will.plugins.devops",
- "will.plugins.friendly",
- "will.plugins.fun",
- "will.plugins.help",
- "will.plugins.productivity",
- "will.plugins.web",
-
- # All plugins in your project.
- "plugins",
-]
-
-# Don't load any of the plugins in this list. Same options as above.
-PLUGIN_BLACKLIST = [
- "will.plugins.productivity.hangout", # Because it requires a HANGOUT_URL
- "will.plugins.productivity.world_time", # Because it requires a WORLD_WEATHER_ONLINE_V2_KEY key
- "will.plugins.productivity.bitly", # Because it requires a BITLY_ACCESS_TOKEN key and the bitly_api library
- "will.plugins.devops.bitbucket_is_up", # Because most folks use github.
- "will.plugins.devops.pagerduty", # Because it requires a PAGERDUTY_SUBDOMAIN and PAGERDUTY_API_KEY key
-]
-
-# ------------------------------------------------------------------------------------
-# Potentially required settings
-# ------------------------------------------------------------------------------------
-
-# If will isn't accessible at localhost, you must set this for his keepalive to work.
-# Note no trailing slash.
-# PUBLIC_URL = "http://my-will.herokuapp.com"
-
-# Port to bind the web server to (defaults to $PORT, then 80.)
-# Set > 1024 to run without elevated permission.
-# HTTPSERVER_PORT = "9000"
-
-
-# ------------------------------------------------------------------------------------
-# Optional settings
-# ------------------------------------------------------------------------------------
-
-# The list of rooms will should join. Default is all rooms.
-# ROOMS = ['Testing, Will Kahuna',]
-
-
-# The room will will talk to if the trigger is a webhook and he isn't told a specific room.
-# Default is the first of ROOMS.
-# DEFAULT_ROOM = 'Testing, Will Kahuna'
-
-
-# Fully-qualified folders to look for templates in, beyond the two that
-# are always included: core will's templates folder, your project's templates folder, and
-# all templates folders in included plugins, if they exist.
-#
-# TEMPLATE_DIRS = [
-# os.path.abspath("other_folder/templates")
-# ]
-
-
-# Access Control: Specify groups of users to be used in the acl=["admins","ceos"] parameter
-# in respond_to and hear actions.
-# Group names can be any string, and the list is composed of user handles.
-# ACL = {
-# "admins": ["steven", "will"]
-# }
-
-
-# Deprecated - please use ACL, above, instead: User handles who are allowed to perform
-# `admin_only` plugins. Defaults to everyone.
-# ADMINS = [
-# "steven",
-# "levi",
-# ]
-
-# Sets a different storage backend. If unset, defaults to redis.
-# If you use a different backend, make sure to add their required settings.
-# STORAGE_BACKEND = "redis" # "redis", "couchbase", or "file".
-
-
-# Disable SSL checks. Strongly reccomended this is not set to True.
-# ALLOW_INSECURE_HIPCHAT_SERVER = False
-
-# Mailgun config, if you'd like will to send emails.
-# DEFAULT_FROM_EMAIL="will@example.com"
-# Set in your environment:
-# export WILL_MAILGUN_API_KEY="key-12398912329381"
-# export WILL_MAILGUN_API_URL="example.com"
-
-
-# Logging level
-# LOGLEVEL = "DEBUG"
-
-# Proxy settings
-# Use proxy to access hipchat servers
-# Make sure your proxy allows CONNECT method to port 5222
-# PROXY_URL = "http://user:pass@corpproxy.example.com:3128"
-# or
-# PROXY_URL = "http://myproxy:80
diff --git a/config.py b/config.py
new file mode 120000
index 00000000..e90cfe37
--- /dev/null
+++ b/config.py
@@ -0,0 +1 @@
+will/scripts/config.py.dist
\ No newline at end of file
diff --git a/docker/.gitignore b/docker/.gitignore
new file mode 100644
index 00000000..f0f18cc1
--- /dev/null
+++ b/docker/.gitignore
@@ -0,0 +1 @@
+*.mike-love
diff --git a/docker/buildwillcontainers.sh b/docker/buildwillcontainers.sh
new file mode 100644
index 00000000..5aa550a3
--- /dev/null
+++ b/docker/buildwillcontainers.sh
@@ -0,0 +1,54 @@
+#!/bin/bash -e
+
+declare -a dockerfiles
+dockerfiles=(
+ ["heywill/will:python2.7$CTAG"]="/will/will-py2/"
+ ["heywill/will:python3.7$CTAG"]="/will/will-py3/")
+
+build_containers() {
+
+ for tag in "${!dockerfiles[@]}";
+ do
+ echo "building $tag with context ${dockerfiles[$tag]}";
+ docker build -t $tag $(dirname $(readlink -f ${BASH_SOURCE[0]}))${dockerfiles[$tag]};
+ echo ""
+ done;
+}
+
+tag_production(){
+ docker tag heywill/will:python2.7$CTAG heywill/will:python2.7
+ echo "tagged heywill/will:python2.7$CTAG as heywill/will:latest"
+
+ docker tag heywill/will:python3.7$CTAG heywill/will:python3.7
+ docker tag heywill/will:python3.7$CTAG heywill/will:latest
+ echo "tagged heywill/will:python3.7$CTAG as heywill/will:latest & heywill/will:python3.7"
+}
+
+
+push_containers(){
+ tag_production
+
+ docker push heywill/will-base:latest
+ docker push heywill/will:latest
+}
+echo "Building with COMMIT TAG: $CTAG"
+case $1 in
+ "--all")
+ build_containers
+ push_containers
+ ;;
+
+ "--build")
+ build_containers
+ ;;
+
+ "--push")
+ push_containers
+ ;;
+
+ *)
+ echo "You did something wrong"
+ exit 1
+ ;;
+esac
+
diff --git a/docker/default.env b/docker/default.env
new file mode 100644
index 00000000..5bd00b34
--- /dev/null
+++ b/docker/default.env
@@ -0,0 +1,20 @@
+
+# REDIS Settings
+WILL_REDIS_URL=redis://redis:6379/
+
+# WILL Settings
+WILL_HTTPSERVER_PORT=8080
+WILL_LOGLEVEL=ERROR
+
+# For Slack
+WILL_SLACK_API_TOKEN
+
+# For Hipchat
+WILL_HIPCHAT_USERNAME
+WILL_HIPCHAT_PASSWORD
+WILL_HIPCHAT_V2_TOKEN
+
+# Rocket.Chat
+WILL_ROCKETCHAT_USERNAME
+WILL_ROCKETCHAT_PASSWORD
+WILL_ROCKETCHAT_URL
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
new file mode 100644
index 00000000..0eacadb7
--- /dev/null
+++ b/docker/docker-compose.yml
@@ -0,0 +1,19 @@
+version: '3.2'
+
+services:
+ redis:
+ image: redis:alpine
+ volumes:
+ - ./data/redis:/data
+ will:
+ image: heywill/will:latest
+ ports:
+ - 8080:8080
+ env_file:
+ - default.env
+ volumes:
+ - ./plugins:/opt/will/plugins
+ - ./templates:/opt/will/templates
+ - ./config.py:/opt/will/config.py
+ depends_on:
+ - redis
diff --git a/docker/will/Dockerfile b/docker/will/Dockerfile
new file mode 100755
index 00000000..ba6a411f
--- /dev/null
+++ b/docker/will/Dockerfile
@@ -0,0 +1,21 @@
+#Pull from will-base
+FROM heywill/will-base:latest
+ARG branch=master
+ARG backends="HipChat Rocket.chat Slack Shell"
+ENV PACKAGES="\
+ dumb-init \
+ bash \
+ ca-certificates \
+ python2 \
+ py-setuptools \
+ libffi-dev \
+"
+# Maintainer
+# ----------
+LABEL maintainer=mlove@columnit.com
+
+RUN pip install git+https://github.com/skoczen/will.git@$branch
+
+WORKDIR $_WILL_HOME
+RUN generate_will_project --backends $backends
+CMD $_WILL_HOME/run_will.py
diff --git a/docker/will/will-py2/Dockerfile b/docker/will/will-py2/Dockerfile
new file mode 100755
index 00000000..f101507a
--- /dev/null
+++ b/docker/will/will-py2/Dockerfile
@@ -0,0 +1,21 @@
+#Pull from will-base
+FROM heywill/will-base:2.7-alpine
+ARG repo=https://github.com/skoczen/will.git
+ARG branch=master
+ARG backends="HipChat Rocket.chat Slack Shell"
+ENV PACKAGES="\
+ dumb-init \
+ bash \
+ ca-certificates \
+ python2 \
+ py-setuptools \
+ libffi-dev \
+"
+# Maintainer
+# ----------
+LABEL maintainer=mlove@columnit.com
+
+RUN pip install git+$repo@$branch
+WORKDIR $_WILL_HOME
+RUN generate_will_project --backends $backends
+CMD $_WILL_HOME/run_will.py
diff --git a/docker/will/will-py3/Dockerfile b/docker/will/will-py3/Dockerfile
new file mode 100755
index 00000000..fc039f90
--- /dev/null
+++ b/docker/will/will-py3/Dockerfile
@@ -0,0 +1,21 @@
+#Pull from will-base
+FROM heywill/will-base:3.7-alpine
+ARG repo=https://github.com/skoczen/will.git
+ARG branch=master
+ARG backends="HipChat Rocket.chat Slack Shell"
+ENV PACKAGES="\
+ dumb-init \
+ bash \
+ ca-certificates \
+ python2 \
+ py-setuptools \
+ libffi-dev \
+"
+# Maintainer
+# ----------
+LABEL maintainer=mlove@columnit.com
+
+RUN pip install git+https://github.com/skoczen/will.git@$branch
+WORKDIR $_WILL_HOME
+RUN generate_will_project --backends $backends
+CMD $_WILL_HOME/run_will.py
diff --git a/docs/backends/analysis.md b/docs/backends/analysis.md
new file mode 100644
index 00000000..d0d2233a
--- /dev/null
+++ b/docs/backends/analysis.md
@@ -0,0 +1,68 @@
+
+## Overview
+We never communicate in the void - there's always a context, and things to read into a particular message, depending who said it, when, and how - and that's exactly what Will's analysis backends are for.
+
+They look at an incoming message and everything around it, and add context.
+
+Will has the following analysis backends built-in, more are on the way (like sentiment analysis) and it's easy to make your own or contribute one to the project:
+
+- History (`will.backends.analysis.history`)
+- Nothing (`will.backends.analysis.nothing`)
+
+
+## Choosing your backends
+
+
+Here's a bit more about the built-ins, and when they'd be a good fit:
+
+#### History (`will.backends.analysis.history`)
+
+Just adds the last 20 messages he heard into the context, and stores this one for the future.
+
+*Required settings*: None
+
+
+#### Nothing (`will.backends.analysis.nothing`)
+
+Does absolutely nothing. But it is a nice template for building your own!
+
+*Required settings*: None
+
+
+For the moment, there's no reason not to just include both built-in backends. But as Will grows and additional options are added, these documents will be updated to explain the tradeoffs in enabling or disabling certain backends.
+
+## Setting your backends
+
+To set your analysis backends, just update the following in `config.py`
+
+```python
+# Backends to analyze messages and generate useful metadata
+ANALYZE_BACKENDS = [
+ "will.backends.analysis.nothing",
+ "will.backends.analysis.history",
+]
+```
+
+
+## Contributing a new backend
+
+Writing a new analysis backend is fairly straightforward - simply subclass `BaseStorageBackend`, and implement the do_analysis method:
+
+
+```python
+from will.backends.analysis.base import AnalysisBackend
+
+class NewBackend(AnalysisBackend):
+
+ def do_analyze(self, message):
+ # Do smart stuff
+ return {
+ "smart": "stuff",
+ "cool": "things",
+ }
+
+```
+
+From there, just test it out, and when you're ready, submit a [pull request!](https://github.com/skoczen/will/pulls)
+
+Now we've got context, let's look at how [Will generates possibilities](/backends/generation).
\ No newline at end of file
diff --git a/docs/backends/encryption.md b/docs/backends/encryption.md
new file mode 100644
index 00000000..ae4bed72
--- /dev/null
+++ b/docs/backends/encryption.md
@@ -0,0 +1,58 @@
+# Encryption Backends
+
+## Overview
+Encryption backends are what lets Will keep his thoughts private - safe from prying eyes, and would-be spies.
+
+All of Will's short and long-term memory (`pubsub` and `storage`) are encrypted by default.
+
+Will supports the following options for storage backend, and improvements and more backends are welcome:
+
+- AES (`will.backends.storage.aes`) - uses AES in CBC mode to encrypt.
+## Choosing a backend
+
+Right now, your only option is AES. So go with that! :)
+
+## Setting your backend
+
+To set your backend, in `config.py`, set:
+
+```python
+# Turn on/off encryption in pub/sub and storage (default is on).
+# Causes a small speed bump, but secures messages in an untrusted environment.
+# ENABLE_INTERNAL_ENCRYPTION = True
+ENCRYPTION_BACKEND = "aes"
+```
+
+## Contributing a new backend
+
+Writing a new encryption backend is easy (if you've got the encryption stuff sorted.) Just subclass `BaseStorageBackend`, and implement:
+
+1. `encrypt_to_b64` - a method that take an arbitary python object, and returns an encrypted, base64 string.
+2. `decrypt_from_b64` - a method that takes that base64 string, and returns a python object.
+3. Provide a `bootstrap()` method that returns an instantiated EncryptionClass.
+
+Here's an example:
+
+```python
+from will.backends.storage.base import BaseStorageBackend
+
+
+class MyGreatEncryption(WillBaseEncryptionBackend):
+
+ @classmethod
+ def encrypt_to_b64(cls, raw):
+ return binascii.b2a_base64(my_encryption_method(pickle.dumps(raw, -1)))
+
+ @classmethod
+ def decrypt_from_b64(cls, raw_enc):
+ return pickle_loads(binascii.a2b_base64(my_decryption_method(raw_enc)))
+
+def bootstrap(settings):
+ return MyGreatEncryption(settings)
+
+
+```
+
+From there, just test it out, and when you're ready, submit a [pull request!](https://github.com/skoczen/will/pulls)
+
+That's Will's brain, end-to-end. If you haven't already, dig into how to get him [deployed](/deploy.md)!
\ No newline at end of file
diff --git a/docs/backends/execution.md b/docs/backends/execution.md
new file mode 100644
index 00000000..00e32c03
--- /dev/null
+++ b/docs/backends/execution.md
@@ -0,0 +1,65 @@
+# Execution Backends
+
+## Overview
+After we've thought of all the possibilities, we then have to decide what we want to do or say. That's where Will's execution backends come in.
+
+They take the context created by `analysis`, and the options created by `generation`, and make a decision on what to do.
+
+Will has the following execution backends built-in, and it's easy to make your own or contribute one to the project:
+
+- All (`will.backends.execution.all`)
+- Best Score (`will.backends.execution.best_score`)
+
+
+## Choosing your backends
+
+
+Here's a bit more about the built-ins, and when they'd be a good fit:
+
+
+#### Best Score (`will.backends.execution.best_score`)
+
+This is the right fit for most people, and it's the most similar to how our brains work. Will looks at the options he has, and picks the single one he thinks is the best.
+
+#### All (`will.backends.execution.all`)
+
+
+
+This is Will's crazy, do-everything mode. He'll take every idea he got in the generation cycle and do *all* of them. Why? Because he's crazy like that.
+
+Or, more likely, because you've built a custom generation backend that limits him down to a set of options you always want done.
+
+## Setting your backends
+
+To set your execution backends, just update the following in `config.py`
+
+```python
+# The "decision making" backends that look among the generated choices,
+# and decide which to follow. Backends are executed in order, and any
+# backend can stop further evaluation.
+EXECUTION_BACKENDS = [
+ "will.backends.execution.best_score",
+ # "will.backends.execution.all",
+]
+```
+
+
+## Contributing a new backend
+
+Writing a new execution backend is reasonably straightforward - simply subclass `ExecutionBackend`, and implement `handle_execution`, making sure to call `self.execute(option)` for the option(s) you choose.:
+
+```python
+from will.backends.execution.base import ExecutionBackend
+
+
+class MyRandomExecutionBackend(ExecutionBackend):
+
+ def handle_execution(self, message):
+ random_option = random.choice(message.generation_options)
+ self.execute(random_option)
+
+```
+
+From there, just test it out, and when you're ready, submit a [pull request!](https://github.com/skoczen/will/pulls)
+
+That's it for Will's decision making. If you'd like a little more esoteric deep-dive, let's look at how his [short-term memory (pubsub)](/backends/pubsub).
\ No newline at end of file
diff --git a/docs/backends/generation.md b/docs/backends/generation.md
new file mode 100644
index 00000000..98306fce
--- /dev/null
+++ b/docs/backends/generation.md
@@ -0,0 +1,84 @@
+# Generation Backends
+
+## Overview
+Generation backends are Will's equivalent of that momement when we pause, and run through all the possible things we could say. Some of them are great ideas. Some of them are terrible ideas. But the process of _generation_ doesn't care - it's just about making as many ideas as possible.
+
+Will's generation backends do the same thing - try to come up with things Will *could* say or do to respond to what he heard.
+
+Will has the following generation backends built-in, and it's easy to add your own or contribute one to the project:
+
+- Fuzzy Match (all) (`will.backends.generation.fuzzy_all_matches`)
+- Fuzzy Match (best) (`will.backends.generation.fuzzy_best_match`)
+- Strict Regex (`will.backends.generation.strict_regex`)
+
+
+## Choosing your backends
+
+Like our brain processes, we can have lots of different ways to generate ideas, working together. You don't have to pick just one generation backend for Will. Depending on your setup, it might be the more, the merrier.
+
+Here's a bit more about the built-ins, and when they'd be a good fit:
+
+#### Fuzzy Match (all) (`will.backends.generation.fuzzy_all_matches`)
+
+This uses the fantastic [fuzzywuzzy](https://github.com/seatgeek/fuzzywuzzy) library to match strings with some fuzziness, as specified by `FUZZY_MINIMUM_MATCH_CONFIDENCE` (defaults to 90% confidence) and `FUZZY_REGEX_ALLOWABLE_ERRORS` (defaults to 3).
+
+Great if you'd like your Will to be a little flexible, sometimes get things wrong, but to handle typos.
+
+*Required settings*: `FUZZY_MINIMUM_MATCH_CONFIDENCE` and `FUZZY_REGEX_ALLOWABLE_ERRORS`
+
+#### Fuzzy Match (best) (`will.backends.generation.fuzzy_best_match`)
+
+This backend is very similar to `fuzzy_all_matches`, but instead of returning all matches above a certain confidence, it just returns the best one, regardless of how good it is.
+
+In general, there's no reason to use this over `fuzzy_all_matches`, unless you have a specific scenario that means you always want a response, but can't be sure of a confidence level.
+
+#### Strict Regex (`will.backends.generation.strict_regex`)
+
+Great for exact matches only. If you only want your Will to do thing when it hears an exact command, or you have a bunch of different commands you're worried about getting mixed up in the fuzziness, `strict_regex` is the way for you to go.
+
+This is the same behavior that was in Will 1.x and 0.x.
+
+## Setting your backends
+
+To set your generation backends, just update the following in `config.py`
+
+```python
+# Backends to generate possible actions, and metadata about them.
+GENERATION_BACKENDS = [
+ "will.backends.generation.fuzzy_all_matches",
+ "will.backends.generation.strict_regex",
+ # "will.backends.generation.fuzzy_best_match",
+]
+```
+
+
+## Contributing a new backend
+
+Writing a new generation backend is easy - just subclass `GenerationBackend`, and implement `do_generate`:
+
+Note that the method should return a list of `GeneratedOption`s, including context, the backend name, and a score.
+
+
+```python
+from will.backends.generation.base import GenerationBackend, GeneratedOption
+
+
+class MyGreatGenerationBackend(GenerationBackend):
+
+ def do_generate(self, event):
+ """Returns a list of GeneratedOptions"""
+ matches = []
+
+ message = event.data
+ for name, l in self.bot.message_listeners.items():
+ if this_is_a_perfect_match(message, l):
+ o = GeneratedOption(context=context, backend="regex", score=100)
+ matches.append(o)
+
+ return matches
+
+```
+
+From there, just test it out, and when you're ready, submit a [pull request!](https://github.com/skoczen/will/pulls)
+
+Now we've got a host of possible things Will can do and say. It's time to look at how [Will decides what to do](/backends/execution).
\ No newline at end of file
diff --git a/docs/backends/io.md b/docs/backends/io.md
new file mode 100644
index 00000000..7e582e9e
--- /dev/null
+++ b/docs/backends/io.md
@@ -0,0 +1,83 @@
+# IO Backends
+
+## Overview
+IO backends are how Will talks and listens to the outside world. They're designed to abstract away the technical intracies of interfacing with a given platform, and let users just _use_ them.
+
+Will supports the following io backends:
+
+- Slack (`will.backends.storage.slack`)
+- Hipchat (`will.backends.storage.hipchat`)
+- Rocket.Chat (`will.backends.storage.rocketchat`)
+- Shell (`will.backends.storage.shell`)
+
+
+## Choosing your backends
+
+This isn't a zero-sum game with will. You can use as many backends as you'd like, all at once.
+
+
+## Setting your backends
+
+To set your io backends, just update the following in `config.py`
+
+```python
+# Platforms and mediums messages can come in and go out on.
+IO_BACKENDS = [
+ "will.backends.io_adapters.slack",
+ "will.backends.io_adapters.hipchat",
+ "will.backends.io_adapters.rocketchat",
+ "will.backends.io_adapters.shell",
+]
+```
+
+## Implementing a new backend
+
+Writing a new storage backend is fairly straightforward - simply subclass `BaseStorageBackend`, and implement:
+
+1. the five required methods, then
+2. specify any required settings with `required_settings`.
+
+
+```python
+from will.backends.storage.base import BaseStorageBackend
+
+
+class MyCustomStorageBackend(BaseStorageBackend):
+ """A custom storage backend using the latest, greatest technology.
+
+ You'll need to provide a GREAT_API_KEY to use it.
+
+ """"
+
+ required_settings = [
+ {
+ "name": "GREAT_API_KEY",
+ "obtain_at": """1. Go to greatamazingtechnology.com/api
+2. Click "Generate API Key"
+3. Copy that key, and set it in your Will.
+""",
+ },
+ ]
+
+
+ # All storage backends must supply the following methods:
+ def __init__(self, *args, **kwargs):
+ # Connects to the storage provider.
+
+ def do_save(self, key, value, expire=None):
+ raise NotImplemented
+
+ def do_load(self, key):
+ raise NotImplemented
+
+ def clear(self, key):
+ raise NotImplemented
+
+ def clear_all_keys(self):
+ raise NotImplemented
+
+```
+
+From there, just test it out, and when you're ready, submit a [pull request!](https://github.com/skoczen/will/pulls)
+
+Now that you've got hearing and talking sorted, let's look at how [Will adds context](/backends/analysis).
\ No newline at end of file
diff --git a/docs/backends/overall.md b/docs/backends/overall.md
new file mode 100644
index 00000000..00ffc50d
--- /dev/null
+++ b/docs/backends/overall.md
@@ -0,0 +1,20 @@
+# Backend Overview
+
+Will is designed rather a lot like our own brains.
+
+His backends are specific parts of the process of noticing something, deciding what to do, and taking action, and they each interact with each other.
+
+Just like us, he hears and sees things (`io`), understands the context in which they happened (`analysis`), considers possible actions (`generation`), decides on something to do (`execution`), and does it (`io`). Along the way, he has both short term, working memory that lets these processes talk to each other (`pubsub`), and long-term, permanent memory (`storage`).
+
+If you've been thinking about diving into Artificial Intelligence or bots, and wondering how it works, the answer lies right there, in your own head.
+
+Below, you can dig a little deeper, see how each piece of Will's brain works, what things you should think about as you set him up, and how you can make him even better.
+
+- [Noticing and Acting `io`](/backends/io)
+- [Analyzing and Context `analysis`](/backends/analysis)
+- [Generating possible actions `generation`](/backends/generation)
+- [Deciding what to do `execution`](/backends/execution)
+- [Short-term, working memory `pubsub`](/backends/pubsub)
+- [Long term memory `storage`](/backends/storage)
+
+
diff --git a/docs/backends/pubsub.md b/docs/backends/pubsub.md
new file mode 100644
index 00000000..31a5ad10
--- /dev/null
+++ b/docs/backends/pubsub.md
@@ -0,0 +1,82 @@
+# Publish-Subscribe (Pubsub) Backends
+
+## Overview
+Pubsub backends handle all the internal messaging between Will's core components. They're designed to be lightweight, reliable, and ephemeral - a lot like our brain's working memory.
+
+Will supports the following options for pubsub backend:
+
+- Redis (`will.backends.pubsub.redis`)
+
+## Choosing a backend
+
+Right now, you're stuck with Redis. :)
+
+## Setting your backends
+
+To set your pubsub backend, just update the following in `config.py`
+
+```python
+PUBSUB_BACKEND = "redis" # "redis", or "zeromq" (beta).
+```
+
+
+## Contributing a new backend
+
+Writing a new pubsub backend is fairly straightforward - simply subclass `BasePubSub`, and implement:
+
+1. the four required methods, and
+2. a bootstrap method.
+
+```python
+from will.backends.pubsub.base import BasePubSub
+
+
+class MyCustomPubsubBackend(BasePubSub):
+ """A custom pubsub backend using the latest, greatest framework.
+
+ You'll need to provide a GREAT_API_KEY to use it.
+
+ """"
+ required_settings = [
+ {
+ "name": "GREAT_API_KEY",
+ "obtain_at": """1. Go to greatamazingframework.com/api
+2. Click "Generate API Key"
+3. Copy that key, and set it in your Will.
+""",
+ },
+ ]
+
+ def __init__(self, settings):
+ # Do whatever I need to do to kick off the backend.
+
+ def do_subscribe(self, topic):
+ """
+ Registers with the backend to only get messages matching a specific topic.
+ Where possible, wildcards are allowed
+ """
+ raise NotImplementedError
+
+ def do_unsubscribe(self, topic):
+ """Unregisters with the backend for a given topic."""
+ raise NotImplementedError
+
+ def publish_to_backend(self, topic, str):
+ """Publishes a string to the backend with a given topic."""
+ raise NotImplementedError
+
+ def get_from_backend(self):
+ """
+ Gets the latest pending message from the backend (FIFO).
+ Returns None if no messages are pending, and is expected *not* to be blocking.
+ """
+ raise NotImplementedError
+
+def bootstrap(settings)
+ MyCustomPubsubBackend(settings)
+
+```
+
+From there, just test it out, and submit a [pull request!](https://github.com/skoczen/will/pulls)
+
+That's it for Will's pubsub backends. He can also remember things long-term. For that, read up on his [long-term memory (storage)](/backends/storage).
\ No newline at end of file
diff --git a/docs/backends/storage.md b/docs/backends/storage.md
new file mode 100644
index 00000000..15c9b5b3
--- /dev/null
+++ b/docs/backends/storage.md
@@ -0,0 +1,79 @@
+# Storage Backends
+
+## Overview
+Storage backends handle all of Will's long-term memory. They're designed to be durable, reliable, and robust - a lot like our brain's long-term memory, but without the forgetfulness.
+
+Will supports the following options for storage backend:
+
+- Redis (`will.backends.storage.redis`)
+- Couchbase (`will.backends.storage.couchbase`)
+- File (`will.backends.storage.file`)
+
+## Choosing a backend
+
+In general, we recommend using Redis, especially since for v2.0, it's also required for pubsub to get Will working. However, in the future, we'll have more pubsub options, and this will be a more option choice.
+
+If you're running in an environment with no access to external resources or ability to install packages, the `File` backend will get you sorted. If you're already running Couchbase for various reasons, it might make the most sense to use it.
+
+But for the moment, for most configurations, we recommend Redis. It's stable, fast, well-supported, and just works.
+
+
+## Setting your backends
+
+To set your storage backend, just update the following in `config.py`
+
+```python
+STORAGE_BACKEND = "redis" # "redis", "couchbase", or "file".
+```
+
+## Contributing a new backend
+
+Writing a new storage backend is fairly straightforward - simply subclass `BaseStorageBackend`, and implement:
+
+1) the five required methods, then
+2) specify any required settings with `required_settings`.
+
+
+```python
+from will.backends.storage.base import BaseStorageBackend
+
+
+class MyCustomStorageBackend(BaseStorageBackend):
+ """A custom storage backend using the latest, greatest technology.
+
+ You'll need to provide a GREAT_API_KEY to use it.
+
+ """"
+
+ required_settings = [
+ {
+ "name": "GREAT_API_KEY",
+ "obtain_at": """1. Go to greatamazingtechnology.com/api
+2. Click "Generate API Key"
+3. Copy that key, and set it in your Will.
+""",
+ },
+ ]
+
+
+ # All storage backends must supply the following methods:
+ def __init__(self, *args, **kwargs):
+ # Connects to the storage provider.
+
+ def do_save(self, key, value, expire=None):
+ raise NotImplemented
+
+ def do_load(self, key):
+ raise NotImplemented
+
+ def clear(self, key):
+ raise NotImplemented
+
+ def clear_all_keys(self):
+ raise NotImplemented
+
+```
+
+From there, just test it out, and when you're ready, submit a [pull request!](https://github.com/skoczen/will/pulls)
+
+That's all you need to know to tweak and improve Will's memory. There's just one topic left in his brain - keeping things private with [encryption](/backends/encryption).
\ No newline at end of file
diff --git a/docs/config.md b/docs/config.md
index 161bed21..a908c43b 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -8,11 +8,15 @@ All environment variables prefixed with `WILL_` are imported into will's `settin
In best practices, you should keep all of the following in environment variables:
-- `WILL_USERNAME`
-- `WILL_PASSWORD`
+- `WILL_SLACK_API_TOKEN`
+- `WILL_HIPCHAT_USERNAME`
+- `WILL_HIPCHAT_PASSWORD`
+- `WILL_HIPCHAT_V2_TOKEN`
+- `WILL_HIPCHAT_V1_TOKEN`
+- `WILL_ROCKETCHAT_USERNAME`
+- `WILL_ROCKETCHAT_PASSWORD`
+- `WILL_ROCKETCHAT_URL`
- `WILL_REDIS_URL`
-- `WILL_V2_TOKEN`
-- `WILL_V1_TOKEN`
- Any other tokens, keys, passwords, or sensitive URLS.
We've made it easy. No excuses. :)
@@ -23,19 +27,32 @@ Config.py is where all of your non-sensitive settings should go. This includes
- `PLUGINS`: The list of plugins to run,
- `PLUGIN_BLACKLIST`: The list of plugins to ignore, even if they're in `PLUGINS`,
+- `IO_BACKENDS`: The list services you want Will to connect to,
+- `ANALYZE_BACKENDS`: The list of message-analysis backends you want Will to run through.
+- `GENERATION_BACKENDS`: The list of reply-generation backends you want Will to go through.
+- `EXECUTION_BACKENDS`: The list of decision-making and execution backends you want Will to go through (we recommend just having one.)
+- `STORAGE_BACKEND`: Which backend you'd like to use for Will to store his long-term memory. (Built-in: 'redis', 'couchbase', 'file')
+- `PUBSUB_BACKEND`: Which backend you'd like to use for Will to use for his working memory. (Built-in: 'redis'. Soon: 'zeromq', 'builtin')
+- `ENCYPTION_BACKEND`: Which backend you'd like to use for Will to encrypt his storage and memory. (Built-in: 'aes'.)
- `PUBLIC_URL`: The publicly accessible URL will can reach himself at (used for [keepalive](plugins/bundled.md#administration)),
- `HTTPSERVER_PORT`: The port will should handle HTTP requests on. Defaults to 80, set to > 1024 if you don't have sudo,
- `REDIS_MAX_CONNECTIONS`: The maximum number of connections to make to redis, for connection pooling.
-- `ROOMS`: The list of rooms to join,
-- `DEFAULT_ROOM`: The room to send messages that come from web requests to,
+- `FUZZY_MINIMUM_MATCH_CONFIDENCE`: What percentage of confidence Will should have before replying to a fuzzy match.
+- `FUZZY_REGEX_ALLOWABLE_ERRORS`: The maximum number of letters that can be wrong in trying to make a fuzzy match.
+- `SLACK_DEFAULT_CHANNEL`: The default Slack channel to send messages to (via webhooks, etc)
+- `HIPCHAT_ROOMS`: The list of rooms to join,
+- `HIPCHAT_DEFAULT_ROOM`: The room to send messages that come from web requests to,
+- `DEFAULT_BACKEND`: The service to send messages that come from web requests to,
- `TEMPLATE_DIRS`: Extra directories to look for templates,
- `ADMINS`: The mention names of all the admins,
- `LOGLEVEL`: What logging level to use,
- `HIPCHAT_SERVER`: if you're using the [HipChat server beta](https://www.hipchat.com/server), the hostname of the server,
- `ALLOW_INSECURE_HIPCHAT_SERVER`: the option to disable SSL checks (seriously, don't),
+- `ENABLE_INTERNAL_ENCRYPTION`: the option to turn off internal encryption (not recommended, but you can do it.)
- `PROXY_URL`: Proxy server to use, consider exporting it as `WILL_PROXY_URL` environment variable, if it contains sensitive information
- and all of your non-sensitive plugin settings.
+
More expansive documentation on all of those settings is in `config.py`, right where you need it.
## How environment variables and config.py are combined
@@ -57,7 +74,7 @@ The rules for combining are fairly straightforward:
3. Some smart defaulting happens inside settings.py for important variables. For the moment, I'm going to leave that out of the docs, and refer you to `settings.py` as I *believe* things should Just Work, and most people should never need to care. If this decision's wrong, please open an issue, and these docs will be improved!
-That's it for config. Now, let's get your will [deployed](deploy.md)!
+That's it for config. Now, you can either do a deeper dive into [Will's brain](/backends/overall.md), or just get your will [deployed](deploy.md)!
diff --git a/docs/deploy.md b/docs/deploy.md
index 4b9d19fb..f2734f5f 100644
--- a/docs/deploy.md
+++ b/docs/deploy.md
@@ -9,7 +9,7 @@ Heroku is our recommended platform for deployment because it's simple, easy, and
#### Step 1: Set up your heroku app, and a redis addon.
Assuming you have the [heroku toolbelt](https://toolbelt.heroku.com/) installed and all set up, it's as easy as this:
-
+
```bash
heroku create our-will-name
heroku addons:add rediscloud
@@ -26,15 +26,18 @@ At minimum, that's
```bash
heroku config:set \
WILL_PUBLIC_URL="http://our-will-name.herokuapp.com" \
-WILL_USERNAME='12345_123456@chat.hipchat.com' \
-WILL_PASSWORD='asj2498q89dsf89a8df' \
-WILL_V2_TOKEN='asdfjl234jklajfa3azfasj3afa3jlkjiau' \
+# Slack
+WILL_SLACK_API_TOKEN="lkasjflkaklfjlasfjal1249814"
+# Hipchat
+WILL_HIPCHAT_USERNAME='12345_123456@chat.hipchat.com' \
+WILL_HIPCHAT_PASSWORD='asj2498q89dsf89a8df' \
+WILL_HIPCHAT_V2_TOKEN='asdfjl234jklajfa3azfasj3afa3jlkjiau' \
+# Rocket.Chat
+WILL_ROCKETCHAT_USERNAME='will@heywill.io'\
+WILL_ROCKETCHAT_PASSWORD='12o312938asfjilasdlfjasdlkfj'\
+WILL_ROCKETCHAT_URL='https://heywill.rocket.chat'\
```
-If you have more than 30 chat rooms, you must also set the V1 token to avoid hipchat rate limits:
-```bash
-heroku config:set WILL_V1_TOKEN='kjadfj89a34878adf78789a4fae3'
-```
Finally, for will's schedule to be correct, you need to set him to the time zone you want:
@@ -75,6 +78,73 @@ git push heroku
Simple. For best-practices, see our continuous deployment recommendations below.
+## Deploy in Docker
+Will is packaged with a Dockerfile and docker-compose files to allow deploying in a container with redis.
+
+### Pre-requisites
+You should have docker already installed; additionally, the instructions require docker-compose. If you're not using docker-compose, you can still also the [pre-defined images from Docker Hub](https://hub.docker.com/r/heywill/will/).
+
+### Step 1: Configure your container's environment variables
+In Will's docker directory, update the default.env file with your environment's settings. At a minimum, this should contain:
+```bash
+# For Slack
+WILL_SLACK_API_TOKEN
+# For Hipchat
+WILL_HIPCHAT_USERNAME
+WILL_HIPCHAT_PASSWORD
+WILL_HIPCHAT_V2_TOKEN
+# Rocket.Chat
+WILL_ROCKETCHAT_USERNAME
+WILL_ROCKETCHAT_PASSWORD
+WILL_ROCKETCHAT_URL
+```
+Note, we've pre-defined the redis url and the HTTP Server port; if you update these values, make sure you update the docker-compose file accordingly.
+
+### Step 2: Deploy your plugins and templates
+If you have any custom templates, create directories for your plugins and templates, and load your plugins and templates they'll be mounted when the container starts up giving your containerized bot access to your templates.
+```bash
+mkdir ./plugins ./templates
+```
+
+### Step 3: Build will.
+Run build your docker image locally.
+```bash
+docker-compose build
+```
+
+### Step 4: Start the container
+Start your image from docker-compose using:
+```bash
+docker-compose up
+```
+or to run the container in the background,
+```bash
+docker-compose up -d
+```
+
+### Roll your Own Container
+Will docker file(s) are part of the main repository; this lets developers/teams
+build on the containers to embed configuration files, custom plugins etc... For
+now Will containers are represented in python 2.7 and python 3 flavors.
+
+##### Will-base
+Will is built on the alpine distribution(s) of python docker images. This was done
+to provide the smallest container footprint; yet, nothing is perfect so a few
+things need to be added. Will-base provides the foundation to perform these base
+modifications.
+
+##### Will
+The heywill/will image is the container with Will executing as the
+container process. A build args are available for branch based builds from a
+repository. To build from a specific repo and branch use,
+```
+--build-arg repo=
+
+
+# Meet Will
+
+Will is the friendliest, easiest-to-teach bot you've ever used. He works on hipchat, in rooms and 1-1 chats.
+
+He makes teaching your chat bot this simple:
+
+```
+@respond_to("hi")
+def say_hello(self, message):
+ self.say("oh, hello!")
+```
+
+Lots of batteries are included, and you can get your own will up and running in a couple of minutes.
+
+Will started by [Steven Skoczen](http://www.inkandfeet.com), and has been [contributed to by lots of folks](improve.md#shoulders).
+
+Check out the quickstart below!
+
+
+
+# Quickstart
+
+Here's how to set up your system and get will running. If you already write python, it'll probably take less than 5 minutes.
+
+---
+
+## Install prerequisites
+
+Will doesn't need much, just python and a place to store things.
+
+Will can store stuff in Redis, Couchbase, or local storage. Our recommended backend is redis, and we'll describe getting it set up below. [Information on using Couchbase or local storage is here](deploy.md#Storage-Backends).
+
+#### Install redis > 2.4
+
+Official documentation is at [redis.io](http://redis.io/).
+
+If you're on Mac OS X, and using [homebrew](http://brew.sh/), you can simply:
+
+```bash
+brew install redis
+```
+
+On a Redhat (RHEL, Centos, Fedora) machine you can:
+
+```bash
+sudo yum install redis
+sudo service redis enable
+sudo service redis start
+```
+
+On a Debian (Ubuntu, Mint, KNOPPIX) machine to properly install follow the [Redis Quickstart](http://redis.io/topics/quickstart). But you can start more quickly with:
+
+```bash
+sudo apt-get install redis-server
+redis-server
+```
+
+#### Install python > 2.6
+
+Most modern operating systems (Mac OS X, Linux, BSDs, etc) ship with python installed, but if you don't have it, all the info is at [python.org](https://www.python.org/).
+
+#### Install virtualenv
+
+Virtualenv is a tool that lets you keep different python projects separate. It is highly recommended for will (and all other python development!)
+
+The python guide has [a great tutorial on virtualenv](http://docs.python-guide.org/en/latest/dev/virtualenvs/), if you don't already have it running. I'd recommend installing the excellent `virtualenvwrapper` library it mentions as well.
+
+#### Set up a virtualenv for will
+
+If you are using virtualenv wrapper:
+
+```bash
+$ mkproject my_will
+# ... some output, setting up the virtualenv
+$ workon my_will
+(my_will) $
+```
+
+You're now all ready to install will!
+
+
+
+## Get will running locally
+
+#### Setting up the project
+
+Installing will is simple easy. Ready? Go!
+
+From your virtualenv and the folder you want to set up your will in,
+
+```bash
+(my_will) $ pip install will
+# ... output from a bunch of pip libraries installing
+
+(my_will) $ generate_will_project
+# ... output from will making your new project
+
+(my_will) $ ./run_will.py
+# .. the magic begins
+```
+
+That's it!
+
+Note that the first time you run, you'll probably be missing some configuration. That's OK - `run_will` will check your environment, and walk you through getting and setting any necessary config. We'll go through the nitty-gritty later, but if you have any odd setup problems, look in `config.py` - that's where all of the non-sensitive data is stored.
+
+
+This is totally normal output.
+
+Eventually, you'll reach this screen of joy. Now, it's time to play!
+
+
+
+#### Testing will out
+
+Once your will is up and running, hop into any of your hipchat rooms, and say hello!
+
+`@will hi`
+
+
+
+`@will help`
+
+
+
+You're up and running - now it's time to [teach your will a few things](plugins/basics.md)!
+
diff --git a/docs/plugins/basics.md b/docs/plugins/basics.md
index bb1cc385..33de941b 100644
--- a/docs/plugins/basics.md
+++ b/docs/plugins/basics.md
@@ -37,7 +37,7 @@ class BonjourPlugin(WillPlugin):
@respond_to("bonjour")
def say_bonjour_will(self, message):
"""bonjour: I know how to say bonjour! In French!"""
- self.reply(message, "bonjour!")
+ self.reply("bonjour!")
```
diff --git a/docs/plugins/builtins.md b/docs/plugins/builtins.md
index 4ac55a86..2fb03c43 100644
--- a/docs/plugins/builtins.md
+++ b/docs/plugins/builtins.md
@@ -12,6 +12,8 @@ It's as simple as:
```python
self.save("my_key", "my_value")
self.load("my_key", "default value")
+self.append("my_key", "value")
+self.pop("my_key", "value")
```
You can also save a value temporarily by setting the number of seconds before it expires:
@@ -27,11 +29,12 @@ Will includes [Jinja](http://jinja.pocoo.org/) for powerful awesome template ren
```python
-self.rendered_template(template_name, context={})
+self.rendered_template(template_name, context={}, custom_filters=[])
```
- **`template_name`**: path to the template, relative to the `TEMPLATE_DIRS` specified in `config.py`.
- **`context`**: a dictionary to render the template with.
+- **`custom_filters`**: [custom filters](http://jinja.pocoo.org/docs/2.9/api/#custom-filters) support for rendering templates
You can use `rendered_template()` directly in a plugin,
@@ -70,7 +73,7 @@ class BonjourPlugin(WillPlugin):
@respond_to("bonjour")
def say_bonjour_will(self, message):
"""bonjour: I know how to say bonjour! In French!"""
- self.reply(message, "bonjour!")
+ self.reply("bonjour!")
```
@@ -143,7 +146,7 @@ class BonjourPlugin(WillPlugin):
@respond_to("bonjour")
def say_bonjour_will(self, message):
- self.reply(message, settings.HELLO_MESSAGE)
+ self.reply(settings.HELLO_MESSAGE)
```
You can also mark one or more settings as required for your plugin with the `require_settings` decorator, and they'll be checked on startup.
@@ -156,7 +159,7 @@ class BonjourPlugin(WillPlugin):
@require_settings("HELLO_MESSAGE", "ANOTHER_SETTING")
@respond_to("bonjour")
def say_bonjour_will(self, message):
- self.reply(message, settings.HELLO_MESSAGE)
+ self.reply(settings.HELLO_MESSAGE)
```
When will starts up, he'll make sure they've been set:
@@ -167,7 +170,7 @@ When will starts up, he'll make sure they've been set:
## Getting a room's history
-Sometimes you'll want to retrieve a room's history. No problem - get the room's object, and the last 75 messages are sitting on `.history`.
+Sometimes you'll want to retrieve a room's history. No problem - get the room's object, and the last 20 messages are sitting on `analysis.history`.
```python
class HistoryPlugin(WillPlugin):
@@ -175,37 +178,10 @@ class HistoryPlugin(WillPlugin):
@respond_to("^get last message")
def get_history(self, message):
room = self.get_room_from_message(message)
- self.reply(message, room.history[-1]["message"])
+ self.reply(room.analysis["history"][-1])
```
-`.history` is pretty much what's returned from the [HipChat room history API](https://www.hipchat.com/docs/apiv2/method/view_room_history) - the lone exception is that the date has been converted to a python datetime.
-```python
- {
- u'from':{
- u'mention_name':u'First Last',
- u'id':xxxx,
- u'links':{
- u'self': u'https://api.hipchat.com/v2/user/xxxx'
- },
- u'name':u'First Last'
- },
- u'date':datetime.datetime(2015, 1, 26, 15, 26, 52),
- u'mentions':[
- {
- u'mention_name':u'FirstLast',
- u'id':xyxy,
- u'links':{
- u'self': u'https://api.hipchat.com/v2/user/xyxy'
- },
- u'name':u'First Last'
- }
- ],
- u'message':u'Hi there!',
- u'type':u'message',
- u'id':u'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
- }
-```
## Parse natural time
diff --git a/docs/plugins/bundled.md b/docs/plugins/bundled.md
index 16655ee8..1d2265aa 100644
--- a/docs/plugins/bundled.md
+++ b/docs/plugins/bundled.md
@@ -44,7 +44,7 @@ Provides a couple of methods for listing and updating will's internal chat room
## Devops
-Will is our devops team at GreenKahuna, and in the long term, we plan to abstract and include our stack deployer as a plugin. For the moment, he just includes a couple basics:
+Will has served as the devops team at a couple different companies, but we haven't yet been able to get the IP sorted to abstract and open-source those plugins. So, for the moment, he just includes a couple basics:
#### Emergency Contacts
@@ -193,6 +193,28 @@ Sometimes, a picture is worth a thousand words.
+Image me works out of the box, but may not in future releases (we're using a pretty hacky way to get search results.) If you rely on it, configure `image me` properly by setting the following two variables in `config.py` or [in your environment with the appropriate `WILL_` prefix](http://skoczen.github.io/will/config/). Here are some instructions on where to obtain both.
+
+* `GOOGLE_API_KEY`
+ * Go to the [Google Developers Console](https://console.developers.google.com).
+ * From the menu in the top left, choose `API Manager -> Credentials`.
+ * Click `Create credentials` from the drop down, then `API Key`, that's the key you need here.
+ * Optionally, give the key a suitable human-readable name so it looks nice in the list e.g. `will-api`.
+* `GOOGLE_CUSTOM_SEARCH_ENGINE_ID`
+ * [Setup a custom search engine](https://support.google.com/customsearch/answer/2630963?hl=en) (CSE) if you don't already have one.
+ * You'll need to provide a site for the CSE to search - just input anything, so the wizard can create the CSE.
+ * Type something useful like `Will` in the name field and create the CSE.
+ * Edit the settings for the CSE you just created, and in the `Sites to search` section, change the drop-down to `Search the entire web but emphasize included sites`.
+ * Delete the "anything" site you added two steps ago, to clean it up.
+ * In the `Details` section, click the `Search engine ID` button, and copy the ID which you need here.
+
+#### Gif me
+
+Like image me, but *alive*.
+
+
+
+
#### Remind me
This saves our bacon every day. Timeboxes meetings, helps people remember appointments, and enforces self-control.
@@ -217,10 +239,7 @@ You can also remind others as well.
#### World time
-We're a remote company. Maybe you are too. Or your clients are. Or the light/dark cycle of the world just fascinates you. If any of these are you,
-
-1. Get a free `WORLD_WEATHER_ONLINE_V2_KEY` from [world weather online](http://developer.worldweatheronline.com).
-2. Get the time in pretty much any city on earth. Even our globe-trotting CEO hasn't been able to stump him.
+We're a remote company. Maybe you are too. Or your clients are. Or the light/dark cycle of the world just fascinates you. If any of these are you, just ask Will to get the time in pretty much any city on earth. Even our globe-trotting CEO hasn't been able to stump him.
@@ -240,4 +259,4 @@ Will also includes a home page, so you can fire him, up, browse to his URL, and
-You now know everything about plugins. Maybe you're wondering about the [finer points of config](../config.md)? Or perhaps, you're ready to [deploy your will](../deploy.md)?
\ No newline at end of file
+You now know everything about plugins. Maybe you're wondering about the [finer points of config](../config.md)? Or perhaps, you're ready to [deploy your will](../deploy.md)?
diff --git a/docs/plugins/create.md b/docs/plugins/create.md
index 1d507ccb..d4288176 100644
--- a/docs/plugins/create.md
+++ b/docs/plugins/create.md
@@ -74,7 +74,7 @@ class PingPlugin(WillPlugin):
@respond_to("^ping$")
def ping(self, message):
- self.reply(message, "PONG")
+ self.reply("PONG")
```
diff --git a/docs/plugins/notice.md b/docs/plugins/notice.md
index c3c09523..b7470c17 100644
--- a/docs/plugins/notice.md
+++ b/docs/plugins/notice.md
@@ -71,7 +71,7 @@ def standup(self):
self.say("@all Standup! %s" % settings.WILL_HANGOUT_URL)
```
-Under the hood, `@periodic` uses [apscheduler](http://apscheduler.readthedocs.org/en/latest/cronschedule.html#available-fields) to provide its options, so you can use any of the following as keyword arguments:
+Under the hood, `@periodic` uses [apscheduler](https://apscheduler.readthedocs.io/en/v2.1.2/cronschedule.html#available-fields) to provide its options, so you can use any of the following as keyword arguments:
- **`year`**: 4-digit year number
- **`month`**: month number (1-12)
diff --git a/docs/plugins/reply.md b/docs/plugins/reply.md
index 99e98fa8..632e69dc 100644
--- a/docs/plugins/reply.md
+++ b/docs/plugins/reply.md
@@ -11,22 +11,20 @@ Like any normal person, will can talk to the chat room, or in 1-1 chats. To tal
@respond_to("bonjour")
def say_bonjour_will(self, message):
# Awesome stuff
- self.say("Bonjour!", message=message)
+ self.say("Bonjour!")
```
-Note that we pass `messsage` along. This allows will to route his reply to the correct room. Without it, he'll just speak to the `DEFAULT_ROOM`.
-
`say()` comes with a number of options, including color, html, and ping notify.
```python
-self.say(content, message=None, room=None, html=False, color="green", notify=False)
+self.say(content, channel=None, html=False, color="green", notify=False)
```
- **`content`**: the content you want to send to the room. Any string will do, HTML or plain text.
-- **`message`**: (optional) The incoming message object
-- **`room`**: (optional) The room object (from self.available_rooms) to send the message to.
+- **`channel`**: (optional) The name of the channel or room to send the message to. If not specified, Will is smart, and will just reply in the same channel/room/thread.
+- **`service`**: (optional, rare) The name of the service (i.e. 'slack', 'hipchat', 'rocketchat') you want to send the message on. By default Will replies on the same service you contacted him on.
- **`html`**: if the message is HTML. `True` or `False`.
- **`color`**: (chat room only) the hipchat color to send. "yellow", "red", "green", "purple", "gray", or "random". Default is "green".
- **`notify`**: whether the message should trigger a 'ping' notification. `True` or `False`.
@@ -38,22 +36,21 @@ Sometimes you want will to ping you - that's where @name mentions are great. To
```python
@respond_to("^hi") # Basic
def hi(self, message):
- self.reply(message, "hello, %s!" % message.sender.nick)
+ self.reply("hello, %s!" % message.sender.handle)
```
-Note the order of arguments is different here, and `messsage` is required. All the options:
+All the options:
```python
-self.reply(message, content, html=False, color="green", notify=False)
+self.reply(content, html=False, color="green", notify=False, start_thread=False)
```
-
-- **`message`**: The incoming message object. Required
- **`content`**: the content you want to send to the room. HTML or plain text.
- **`html`**: if the message is HTML. `True` or `False`.
- **`color`**: (chat room only) the hipchat color to send. "yellow", "red", "green", "purple", "gray", or "random". Default is "green".
- **`notify`**: whether the message should trigger a 'ping' notification. `True` or `False`.
+- **`start_thread`**: whether Will should start a new thread, if the backend supports it.
@@ -65,9 +62,11 @@ When will recieves messages from webhooks and HTTP requests, he's still connecte
@route("/ping")
def ping(self):
self.say("PONG!")
+ # or
+ self.say("PONG!", room="ping-pong", service="slack")
```
-If you want to talk to a different room, you can pass in the `room` argument with one of the rooms from `self.available_rooms`.
+If you want to talk to a different room, you can pass in the `channel` with the name of the channel or room you want to talk to. If you have multiple services connected, just pass `service` with the one you want.
## Send an email
@@ -110,13 +109,14 @@ def walkmaster(self):
The options are pretty much the same as `self.say`, with the addition of the `when` parameter.
```python
-self.schedule_say(content, when, message=None, room=None, html=False, color="green", notify=False)
+self.schedule_say(content, when, message=None, channel=None, html=False, color="green", notify=False)
```
- **`content`**: the content you want to send to the room. Any string will do, HTML or plain text.
- **`when`**: when you want the message to be said. Python `datetime` object.
- **`message`**: (optional) The incoming message object
-- **`room`**: (optional) The room object (from self.available_rooms) to send the message to.
+- **`channel`**: (optional) The name of the channel or room to send the message to. If not specified, Will is smart, and will just reply in the same channel/room/thread. You can also pass "ALL_ROOMS" to send the message everywhere, if that's really your thing.
+- **`service`**: (optional, rare) The name of the service (i.e. 'slack', 'hipchat', 'rocketchat') you want to send the message on. By default Will replies on the same service you contacted him on.
- **`html`**: if the message is HTML. `True` or `False`.
- **`color`**: (chat room only) the hipchat color to send. "yellow", "red", "green", "purple", "gray", or "random". Default is "green".
- **`notify`**: whether the message should trigger a 'ping' notification. `True` or `False`.
@@ -139,12 +139,13 @@ def give_us_somethin_to_talk_about(self, message):
Note: you can't set the topic of a 1-1 chat. Will will complain politely. All options:
```python
-self.set_topic(topic, message=None, room=None)
+self.set_topic(topic, message=None, channel=None)
```
- `topic`: The string you want to set the topic to
- `message`: (optional) The incoming message object
-- `room`: (optional) The room object (from self.available_rooms) to send the message to.
+- `channel`: (optional) The name of the channel or room to send the message to. If not specified, Will is smart, and will just set the topic for the same channel/room/thread. You can also pass "ALL_ROOMS" to send the message everywhere, if that's really your thing.
+- `service`: (optional, rare) The name of the service you want to send the message on. By default Will replies on the same service you contacted him on.
diff --git a/docs/releases.md b/docs/releases.md
new file mode 100644
index 00000000..73ccc89d
--- /dev/null
+++ b/docs/releases.md
@@ -0,0 +1,426 @@
+# Releases
+
+#### 2.1.3 - June 13, 2018
+
+Bugfix & feature release that includes:
+
+Bugfixes:
+* Will is now robust against slack disconnects, and automatically handles reconnects, thanks to [https://github.com/Ashex](Ashex)!
+* Will also now doesn't respond to every single message when he joins a new channel. Please send your thank-you cards to [https://github.com/Ashex](Ashex).
+* Makes will more robust at handling incorrect channel names in slack, thanks to [reist](https://github.com/reist).
+* Properly renames `SLACK_DEFAULT_ROOM` to `SLACK_DEFAULT_CHANNEL` (with backwards-compatability) thanks (again!) to [reist](https://github.com/reist).
+* Enable/disable backends from `generate_will_project` now properly puts the comment outside the strong. Thanks to [phiro69](https://github.com/phiro69) for the report.
+
+
+New features:
+* Adds support for slack attachments, thanks to [https://github.com/Ashex](Ashex)
+* Reminders now notify the person making the reminder, thanks to [unicolet](https://github.com/unicolet).
+* `generate_will_project` now supports a `--backends` flag, thanks to [wontonst](https://github.com/wontonst).
+* Will now explicitly notes if he's automagically using `REDIS_URL` to find the redis backend.
+
+#### 2.1.2 - March 30, 2018
+
+Bugfix release that includes:
+
+* Fixes python 2 compatability (`str` instead of `basestring`) for the HipChat adapter, thanks to [wontonst](https://github.com/wontonst).
+
+
+#### 2.1.1 - March 22, 2018
+
+Bugfix release that includes:
+
+* Fixes slack reconnect issues, thanks to [@mattcl](https://github.com/mattcl). Props to [@cmachine](https://github.com/cmachine) for also submitting a fix.
+* Saying "G" will no longer give you a picture of a pug, using the default settings. This is both tragic, and necessary. (Actual fix: adjusted default fuziness settings. If you have the fuzzy backend on, and were seeing the rather hilarious/annoying [#327](https://github.com/skoczen/will/issues/327), set `FUZZY_MINIMUM_MATCH_CONFIDENCE = 91` in your config.py)
+* Programmer help is working again, thanks to [@acommasplice](https://github.com/acommasplice).
+* Fixes word game to work in python 3 thanks to [@netjunki](https://github.com/netjunki), and [ptomkiel-oktawave](https://github.com/ptomkiel-oktawave)'s report.
+* Fixes up chat room rosters in HipChat with rosters > 1000 rooms, thanks to [@ostracon](https://github.com/ostracon)
+* Fixes `get_room_from_message`, thanks to [@TaunoTinits](https://github.com/TaunoTinits)'s fix and [ptomkiel-oktawave](https://github.com/ptomkiel-oktawave)'s report.
+* Fixes an error that could occur on incoming webhooks on hipchat. Thanks to [ptomkiel-oktawave](https://github.com/ptomkiel-oktawave) and others for a report.
+* Fixes Will incorrectly talking to the main slack room, when he's directly addressed in 1-1 with something he doesn't know how to do. Thanks to [netjunki](https://github.com/netjunki) for the report!
+
+
+#### 2.1.0 - November 28, 2017
+
+Planned release that includes:
+
+* Automatic [docker hub builds](https://hub.docker.com/r/heywill/will/) thanks to [@mike-love](https://github.com/mike-love)
+* Upgrades to use base markdownify package, as the proposed changes have been [merged and released](https://github.com/matthewwithanm/python-markdownify/pull/1).
+* New fabric commands to manage docker builds and releases.
+
+
+#### 2.0.2 - November 22, 2017
+
+Bugfix release that fixes:
+
+* Will once again joins all hipchat rooms `HIPCHAT_ROOMS` was not specified. Thanks to [vissree](https://github.com/vissree) for finding and reporting this bug!
+
+#### 2.0.1 - November 21, 2017
+
+Same release as 2.0.1, removes beta tag.
+
+#### 2.0.1beta4 - November 20, 2017
+
+Bugfix release that fixes:
+
+* `color` parameter wasn't working properly in Slack.
+* Fixes up slack escaping, to support `Upgrading to Will 2.0
+ +Will was born four years ago, and 0.x and 1.x lived long, good lives. + +But as the years passed, there were increasing concerns about HipChat lock-in, and it was time to make Will smarter. + +So six months ago, work on Will 2.0 began, and today it's finally here! All of our Wills are free from lock-in, future-proofed, and whole lot smarter. If you're just hearing about this, it's worth a quick read of the [release announcement](https://heywill.io/will2) for a bit more context! + +# The Short Version: Just pip install +While Will has picked up a bunch of features and improvements in 2.0, we've aimed to keep him backwards-compatable with 1.x and 0.x releases. If you weren't using any undocumented internal methods and you're already using redis, upgrading should be as easy as: + +```shell +pip install --upgrade will +``` + +From there, you can just `./run_will.py`, and things should Just Work. + +You will, however, see a lot of output from Will, telling you that some names have changed, and asking you to update them in your `config.py` when you have time. You can either just follow those instructions, or the guide below. + +# The Long Version: + +### 1. Add IO backends. + +If you're planning to continue only using HipChat (until it's replaced by Stride), we'd recommend that you add this to your `config.py`: + +```python +IO_BACKENDS = [ + "will.backends.io_adapters.hipchat", + "will.backends.io_adapters.shell", + # "will.backends.io_adapters.slack", + # "will.backends.io_adapters.rocketchat", +] +``` + +That will enable the HipChat and local shell stdin/stdout backend, for easy testing. + +If you want to also bring your Will into a Slack or Rocket.Chat room in the future, just uncomment that backend and restart! + +### 2. Update the HipChat tokens to be namespaced. + +You'll see this starting up, but when you have time, update your tokens as follows: +(If you're using `WILL_` environment variables, please add the `WILL_` as needed:) + +- `USERNAME` becomes `HIPCHAT_USERNAME` +- `TOKEN` or `V1_TOKEN` become `HIPCHAT_V1_TOKEN` +- `V2_TOKEN` becomes `HIPCHAT_V2_TOKEN` +- `DEFAULT_ROOM` becomes `HIPCHAT_DEFAULT_ROOM` +- `HANDLE` should be removed, as it's now pulled live from the HipChat servers and not used. +- `NAME` should be removed, as it's now pulled live from the HipChat servers and not used. +- `PASSWORD` becomes `HIPCHAT_PASSWORD` +- `ROOMS` becomes `HIPCHAT_ROOMS` + + +### 3. Set up Redis + +At the moment, Redis is the only working pubsub backend, and is required to run Will. So, if you're not already running it, you'll need it for 2.x. + +If this is impossible for your setup, ZeroMQ support is in the works, and we're looking to add a pure-python backend as well in 2.1 or 2.2. Creating a new pubsub backend just requires subclassing `BasePubSub` and implementing four methods - so if someone has a little time to bring those along, a pull request is welcome! + + +### 4. Set your encryption secret key. + +Will now encrypts all messages on the pubsub wire and in storage by default. Without a `SECRET_KEY` set, he'll auto-generate one based on the machine MAC address, but this isn't a perfect solution, and will mean that he can't access his storage if there are hardware changes (or he's running in a virtualized environment that has shifting MACs.) + +Please set `SECRET_KEY` as soon as possible. + +The recommended way is to set it as an environmental variable, `WILL_SECRET_KEY`, in an environment that is secured and you trust. Any string will work, and entropy is good. + + +### 5. Set the new 2.0 settings to your liking. + +Will 2 ships with bunch of new features, and though we've provided sensible defaults, if you'd like, you can update your `config.py` with your preferences. + +The simplest way to see everything is to have Will generate a `config.py.dist` that you can use for comparison: + +```shell +$ generate_will_project --config-dist-only +... +Created a config.py.dist. Open it up to see what's new! +``` + +It's worth reading through the new `config.py`, but here's a few areas specifically worth a look: + +### Platform and Decision-making + +As mentioned above, there are now multiple IO backends and platforms that Will can communicate on. It's also now easy to [write your own](/backends/io), pull requests are very welcome, and more are coming soon. Here's all the options: + +```python +IO_BACKENDS = [ + "will.backends.io_adapters.slack", + "will.backends.io_adapters.hipchat", + "will.backends.io_adapters.rocketchat", + "will.backends.io_adapters.shell", +] +``` + + +Will 2 also comes with pluggable brains - split into Analysis, Generation, and Execution backends. The defaults are solid and behave similarly to Will 1.0 (the only difference is a high-confidence fuzzy matching engine), but if you're interested in making your Will more flexible, or adding more context to his responses, building [custom backends is easy](/backends/analysis). + +Here's all of the options, with the defaults uncommented. It's worth pulling this into your `config.py`. + +```python +# Backends to analyze messages and generate useful metadata +ANALYZE_BACKENDS = [ + "will.backends.analysis.nothing", + "will.backends.analysis.history", +] + +# Backends to generate possible actions, and metadata about them. +GENERATION_BACKENDS = [ + "will.backends.generation.strict_regex", + "will.backends.generation.fuzzy_all_matches", + # "will.backends.generation.fuzzy_best_match", +] + +# The "decision making" backends that look among the generated choices, +# and decide which to follow. Backends are executed in order, and any +# backend can stop further evaluation. +EXECUTION_BACKENDS = [ + "will.backends.execution.best_score", + # "will.backends.execution.all", +] +``` + +There are also a few settings to tweak things like the fuzzy logic. These have sensible defaults, but you can tweak them to your liking. + +```python +# Confidence fuzzy generation backends require before Will responds +# https://pypi.python.org/pypi/fuzzywuzzy +FUZZY_MINIMUM_MATCH_CONFIDENCE = 90 # Defaults to 90% +FUZZY_REGEX_ALLOWABLE_ERRORS = 3 +``` + +### 6. That's it - let us know how it goes! + +That's all you really need to know to flip the switch to Will 2.0. + +As there's a lot of new stuff in this release, it's possible that some bugs have slipped through the cracks. Please submit anything you find, no matter how small, [into the github issue tracker](https://github.com/skoczen/will/issues). We'll be active in fixing things ASAP and helping if you're stuck. + +Thanks for using Will, and for going through the big upgrade! We're excited about what the future holds, and happy to get your bots free from platform lock-in. + +If you'd like to do a deeper dive into what's new, check out the documentation [on Will's new brain](/backends/overall/). \ No newline at end of file diff --git a/fabfile.py b/fabfile.py index 5140f2f6..fa0aa43b 100644 --- a/fabfile.py +++ b/fabfile.py @@ -1,7 +1,7 @@ import os import tempfile -from will import VERSION from fabric.api import * +from will import VERSION SITE_DIR = "site" WHITELIST_DIRS = [".git", ] @@ -10,6 +10,27 @@ SANITY_CHECK_PROJECT_FILES = ["fabfile.py", "setup.py", "mkdocs.yml"] SANITY_CHECK_BUILD_FILES = ["index.html", "js", "css"] +CTAG = os.environ.get("CTAG", "") + +DOCKER_BUILDS = [ + { + "ctagname": "heywill/will:python2.7%s" % CTAG, + "name": "heywill/will:python2.7" % os.environ, + "dir": "will/will-py2/", + }, + { + "ctagname": "heywill/will:python2.7%s" % CTAG, + "name": "heywill/will:latest" % os.environ, + "dir": "will/will-py2/", + }, + { + "ctagname": "heywill/will:python3.7%s" % CTAG, + "name": "heywill/will:python3.7" % os.environ, + "dir": "will/will-py3/", + }, +] +DOCKER_PATH = os.path.join(os.getcwd(), "docker") + def _splitpath(path): path = os.path.normpath(path) @@ -26,6 +47,7 @@ def upload_release(): local("python setup.py sdist upload") +@task def release(): deploy_docs() upload_release() @@ -56,21 +78,55 @@ def deploy_docs(): for root, dirs, files in os.walk(root_dir, topdown=False): for name in files: if name not in WHITELIST_FILES and not any([r in WHITELIST_DIRS for r in _splitpath(root)]): - # print "removing %s" % (os.path.join(root, name)) + # print("removing %s" % (os.path.join(root, name))) os.remove(os.path.join(root, name)) for name in dirs: if name not in WHITELIST_DIRS and not any([r in WHITELIST_DIRS for r in _splitpath(root)]): - # print "removing %s" % (os.path.join(root, name)) - os.rmdir(os.path.join(root, name)) + print("removing %s" % (os.path.join(root, name))) + try: + os.rmdir(os.path.join(root, name)) + except: + # Handle symlinks + os.remove(os.path.join(root, name)) local("cp -rv %s/* ." % tempdir) - with settings(warn_only=True): - result = local("git diff --exit-code") + result = local("git diff --exit-code", warn_only=True) if result.return_code != 0: local("git add -A .") local("git commit -m 'Auto-update of docs: %s'" % last_commit) local("git push") else: - print "No changes to the docs." + print("No changes to the docs.") local("git checkout %s" % current_branch) + + +@task +def docker_build(): + print("Building Docker Images...") + with lcd(DOCKER_PATH): + for b in DOCKER_BUILDS: + local("docker build -t %(ctagname)s %(dir)s" % b) + + +def docker_tag(): + print("Building Docker Releases...") + with lcd(DOCKER_PATH): + for b in DOCKER_BUILDS: + local("docker tag %(ctagname)s %(name)s" % b) + + +def docker_push(): + print("Pushing Docker to Docker Cloud...") + with lcd(DOCKER_PATH): + local("docker login -u $DOCKER_USER -p $DOCKER_PASS") + local("docker push heywill/will:python2.7") + local("docker push heywill/will:python3.7") + local("docker push heywill/will:latest") + + +@task +def docker_deploy(): + docker_build() + docker_tag() + docker_push() diff --git a/mkdocs.yml b/mkdocs.yml index 3bfbbaf3..29a486ee 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,6 +3,7 @@ site_url: http://skoczen.github.io/will site_description: "Will is a simple, beautiful-to-code python hipchat bot" pages: - Get Started: index.md +- Upgrade to 2.0: upgrading_to_2.md - Teach Your Will: - The Basics: plugins/basics.md - What Will Can Notice: plugins/notice.md @@ -11,11 +12,36 @@ pages: - Plugin Builtins: plugins/builtins.md - Bundled Plugins: plugins/bundled.md - Configure: config.md +- "Brain": + - Overview: backends/overall.md + - Noticing and Talking: backends/io.md + - Analysis and Context: backends/analysis.md + - Generation: backends/generation.md + - Decisions and Execution: backends/execution.md + - Pubsub and short-term memory: backends/pubsub.md + - Storage and long-term memory: backends/storage.md + - Encryption and privacy: backends/encryption.md - Deploy: deploy.md - Improve: improve.md +# - Making Will Better and Better: improve.md +- Project: + - Releases: releases.md + - Roadmap: roadmap.md +# - "Will's Brain": +extra_javascript: +- themes/will/js/base.js +- themes/will/js/bootstrap-3.0.3.min.js +- themes/will/js/prettify-1.0.min.js +extra_css: +- css/docs.css +- themes/will/css/base.css +- themes/will/css/bootstrap-custom.min.css +- themes/will/css/font-awesome-4.0.3.css +- themes/will/css/prettify-1.0.css +- themes/will/css/style.css # theme: will site_name: Will theme_dir: "docs/themes/will/" repo_url: https://github.com/skoczen/will/ -copyright: Copyright © 2014, GreenKahuna, Steven Skoczen and the individual contributors. +copyright: Copyright © 2017, Steven Skoczen, Ink and Feet, and the individual contributors. # google_analytics: ['UA-1234567-8', 'skoczen.github.io/will'] diff --git a/profile.txt b/profile.txt new file mode 100644 index 00000000..0515a267 Binary files /dev/null and b/profile.txt differ diff --git a/requirements.base.txt b/requirements.base.txt deleted file mode 100644 index ef1f98d8..00000000 --- a/requirements.base.txt +++ /dev/null @@ -1,17 +0,0 @@ -bottle==0.12.7 -clint==0.3.7 -dill==0.2.1 -dnspython==1.12.0 -natural==0.1.5 -requests==2.4.1 -parsedatetime==1.1.2 -pyasn1==0.1.7 -pyasn1-modules==0.0.5 -sleekxmpp==1.3.1 -APScheduler==2.1.2 -CherryPy==3.6.0 -Jinja2==2.7.3 -Markdown==2.3.1 -MarkupSafe==0.23 -PyYAML==3.10 -pygerduty==0.28 diff --git a/requirements.txt b/requirements.txt index 4f430053..e6f122d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,7 @@ -hiredis==0.1.4 -redis==2.10.3 +hiredis==0.2.0 +redis==2.10.6 --r requirements.base.txt +-r will/requirements/base.txt +-r will/requirements/slack.txt +-r will/requirements/hipchat.txt +-r will/requirements/rocketchat.txt diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..b21f1e3c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,9 @@ +[pytest] +addopts = --cov-report term-missing --cov-config=.coveragerc --cov . + +[aliases] +test = pytest + +[flake8] +ignore = F401,N812,F403,E721,E713,F405,W503,E121,E123,E126,E226,E24,E704,D100,D400,D101,D102,N806,D105,D401,D202,D103,D,E722,E741 +max-line-length = 160 diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index abb63dfc..c1b69fc7 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/env python import os +import sys from setuptools import setup, find_packages from will import __name__ as PACKAGE_NAME from will import VERSION @@ -7,14 +8,46 @@ DESCRIPTION = "A friendly python hipchat bot" ROOT_DIR = os.path.dirname(__file__) SOURCE_DIR = os.path.join(ROOT_DIR) +REQS_DIR = os.path.join(ROOT_DIR, "will", "requirements") -reqs = [] -for req_file in ("requirements.base.txt", "requirements.txt"): - with open(req_file, "r+") as f: +install_requires = [] +dependency_links = [] +with open("requirements.txt", "r+") as f: + for line in f.readlines(): + if line[0] == "-": + continue + install_requires.append(line.strip()) + +for req_file in ["base.txt", "slack.txt", "hipchat.txt", "rocketchat.txt"]: + with open(os.path.join(REQS_DIR, req_file), "r+") as f: for line in f.readlines(): - if line[0] == "-": + if ( + (line.startswith("-") and not line.startswith("-e")) + or line.startswith("#") + ): continue - reqs.append(line.strip()) + + if "-e" in line: + line = line.replace("-e", "") + dependency_links.append(line) + line = line.split("#")[-1].split("=")[-1] + + install_requires.append(line.strip()) + + +tests_require = [ + 'pytest==2.8.3', + 'pytest-cov', + 'pytest-runner', + 'mock' +] + +setup_requires = [] +needs_pytest = set(('pytest', 'test', 'ptr')).intersection(sys.argv) + +if needs_pytest: + setup_requires.append('pytest-runner') + try: import pypandoc @@ -35,10 +68,13 @@ url="https://github.com/skoczen/will", version=VERSION, download_url=['https://github.com/skoczen/will/tarball/%s' % VERSION, ], - install_requires=reqs, + install_requires=install_requires, + dependency_links=dependency_links, + setup_requires=setup_requires, + tests_require=tests_require, packages=find_packages(), include_package_data=True, - keywords=["hipchat", "bot"], + keywords=["chatbot", "bot", "ai", "slack", "hipchat", "rocketchat", "stride"], classifiers=[ "Programming Language :: Python", "License :: OSI Approved :: BSD License", @@ -49,6 +85,12 @@ "Topic :: Internet :: WWW/HTTP", "Topic :: Communications :: Chat", "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Framework :: Robot Framework", + "Framework :: Robot Framework :: Library", + "Framework :: Robot Framework :: Tool", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 3", ], entry_points={ 'console_scripts': ['generate_will_project = will.scripts.generate_will_project:main'], diff --git a/source/Will Logo.psd b/source/Will Logo.psd new file mode 100644 index 00000000..1ce622fd Binary files /dev/null and b/source/Will Logo.psd differ diff --git a/source/will-head-icon-2000.png b/source/will-head-icon-2000.png new file mode 100644 index 00000000..eba3c9cd Binary files /dev/null and b/source/will-head-icon-2000.png differ diff --git a/source/will.fla b/source/will.fla index 0b353d22..fd91bcbf 100644 Binary files a/source/will.fla and b/source/will.fla differ diff --git a/source/will2.fla b/source/will2.fla new file mode 100644 index 00000000..9506fd30 Binary files /dev/null and b/source/will2.fla differ diff --git a/start_dev_will.py b/start_dev_will.py index fe1982d0..96dc7dfd 100755 --- a/start_dev_will.py +++ b/start_dev_will.py @@ -1,6 +1,35 @@ #!/usr/bin/env python +import argparse +import os +import shutil +import sys from will.main import WillBot -if __name__ == '__main__': +parser = argparse.ArgumentParser() +parser.add_argument( + '--profile', + action='store_true', + help='Run with yappi profiling.' +) +args = parser.parse_args() + + +def start_will(): + if args.profile: + try: + import yappi + except: + print("Unable to run Will in profiling mode without yappi. Please `pip install yappi`.") + sys.exit(1) + try: + shutil.rmtree('will_profiles') + except OSError: + pass + os.makedirs("will_profiles") + bot = WillBot() bot.bootstrap() + + +if __name__ == '__main__': + start_will() diff --git a/tox.ini b/tox.ini index b51d939c..8998c475 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,15 @@ -[flake8] -ignore = F401,N812,F403,E721,E713 -max-line-length = 120 -# exclude = tests/* -# max-complexity = 12 \ No newline at end of file +[tox] +envlist = py27, 35, flake8 + + +[testenv] +commands = + python setup.py test +deps = + flake8 + sleekxmpp==1.3.2 + -rrequirements.txt + +[testenv:flake8] +commands = + flake8 \ No newline at end of file diff --git a/will/__init__.py b/will/__init__.py index 9a38c180..4260069c 100644 --- a/will/__init__.py +++ b/will/__init__.py @@ -1 +1 @@ -VERSION = "0.9.3" +VERSION = "2.1.3" diff --git a/will/abstractions.py b/will/abstractions.py new file mode 100644 index 00000000..3ba43c61 --- /dev/null +++ b/will/abstractions.py @@ -0,0 +1,168 @@ +# -- coding: utf-8 - +import datetime +import hashlib +import logging +from pytz import timezone as pytz_timezone + +from will.utils import Bunch + + +class Message(object): + will_internal_type = "Message" + REQUIRED_FIELDS = [ + "is_direct", + "is_private_chat", + "is_group_chat", + "will_is_mentioned", + "will_said_it", + "sender", + "backend_supports_acl", + "content", + "backend", + "original_incoming_event", + ] + + def __init__(self, *args, **kwargs): + for f in self.REQUIRED_FIELDS: + if not f in kwargs: + raise Exception("Missing %s in Message construction." % f) + + for f in kwargs: + self.__dict__[f] = kwargs[f] + + if "timestamp" in kwargs: + self.timestamp = kwargs["timestamp"] + else: + self.timestamp = datetime.datetime.now() + + # Clean content. + self.content = self._clean_message_content(self.content) + + h = hashlib.md5() + h.update(self.timestamp.strftime("%s").encode("utf-8")) + h.update(self.content.encode("utf-8")) + self.hash = h.hexdigest() + + self.metadata = Bunch() + if not "original_incoming_event_hash" in kwargs: + if hasattr(self, "original_incoming_event") and hasattr(self.original_incoming_event, "hash"): + self.original_incoming_event_hash = self.original_incoming_event.hash + else: + self.original_incoming_event_hash = self.hash + + def __unicode__(self, *args, **kwargs): + if len(self.content) > 20: + content_str = "%s..." % self.content[:20] + else: + content_str = self.content + return u"Message: \"%s\"\n %s (%s) " % ( + content_str, + self.timestamp.strftime('%Y-%m-%d %H:%M:%S'), + self.backend, + ) + + def __str__(self, *args, **kwargs): + return self.__unicode__(*args, **kwargs) + + def _clean_message_content(self, s): + # Clear out 'smart' quotes and the like. + s = s.replace("’", "'").replace("‘", "'").replace('“', '"').replace('”', '"') + s = s.replace(u"\u2018", "'").replace(u"\u2019", "'") + s = s.replace(u"\u201c", '"').replace(u"\u201d", '"') + return s + + +class Event(Bunch): + will_internal_type = "Event" + + REQUIRED_FIELDS = [ + "type", + "version", + ] + + def __init__(self, *args, **kwargs): + super(Event, self).__init__(*args, **kwargs) + self.version = 1 + + for f in self.REQUIRED_FIELDS: + if not f in kwargs and not hasattr(self, f): + raise Exception("Missing %s in Event construction." % f) + + if "timestamp" in kwargs: + self.timestamp = kwargs["timestamp"] + else: + self.timestamp = datetime.datetime.now() + + h = hashlib.md5() + h.update(self.timestamp.strftime("%s").encode("utf-8")) + h.update(self.type.encode("utf-8")) + self.hash = h.hexdigest() + if not "original_incoming_event_hash" in kwargs: + if hasattr(self, "original_incoming_event") and hasattr(self.original_incoming_event, "hash"): + self.original_incoming_event_hash = self.original_incoming_event.hash + else: + self.original_incoming_event_hash = self.hash + + +class Person(Bunch): + will_is_person = True + will_internal_type = "Person" + REQUIRED_FIELDS = [ + "id", + "handle", + "mention_handle", + "source", + "name", + "first_name" + # "timezone", + ] + + def __init__(self, *args, **kwargs): + super(Person, self).__init__(*args, **kwargs) + + for f in kwargs: + self.__dict__[f] = kwargs[f] + + # Provide first_name + if "first_name" not in kwargs: + self.first_name = self.name.split(" ")[0] + + for f in self.REQUIRED_FIELDS: + if not hasattr(self, f): + raise Exception("Missing %s in Person construction." % f) + + # Set TZ offset. + if hasattr(self, "timezone") and self.timezone: + self.timezone = pytz_timezone(self.timezone) + self.utc_offset = self.timezone._utcoffset + else: + self.timezone = False + self.utc_offset = False + + @property + def nick(self): + logging.warn("sender.nick is deprecated and will be removed eventually. Please use sender.handle instead!") + return self.handle + + +class Channel(Bunch): + will_internal_type = "Channel" + REQUIRED_FIELDS = [ + "id", + "name", + "source", + "members", + ] + + def __init__(self, *args, **kwargs): + super(Channel, self).__init__(*args, **kwargs) + + for f in self.REQUIRED_FIELDS: + if not f in kwargs: + raise Exception("Missing %s in Channel construction." % f) + for f in kwargs: + self.__dict__[f] = kwargs[f] + + for id, m in self.members.items(): + if not m.will_is_person: + raise Exception("Someone in the member list is not a Person instance.\n%s" % m) diff --git a/will/acl.py b/will/acl.py index a78d6781..1cc48df2 100644 --- a/will/acl.py +++ b/will/acl.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- -from . import settings +import logging +from will import settings def get_acl_members(acl): @@ -9,7 +10,7 @@ def get_acl_members(acl): if getattr(settings, "ACL", None): try: # Case-insensitive checks - for k, v in settings.ACL.items(): + for k in settings.ACL.keys(): if k.lower() == acl: acl_members = settings.ACL[k] break @@ -20,10 +21,41 @@ def get_acl_members(acl): def is_acl_allowed(nick, acl): - nick = nick.lower() + if not getattr(settings, "ACL", None): + logging.warn( + "%s was just allowed to perform actions in %s because no ACL settings exist. This can be a security risk." % ( + nick, + acl, + ) + ) + return True for a in acl: acl_members = get_acl_members(a) - if nick in acl_members: + if nick in acl_members or nick.lower() in [x.lower() for x in acl_members]: return True return False + + +def test_acl(message, acl): + try: + if settings.DISABLE_ACL: + return True + + allowed = is_acl_allowed(message.sender.handle, acl) + if allowed: + return True + if hasattr(message, "data") and hasattr(message.data, "backend_supports_acl"): + if not message.data.backend_supports_acl: + logging.warn( + "%s was just allowed to perform actions in %s because the backend does not support ACL. This can be a security risk." % ( + message.sender.handle, + acl, + ) + + "To fix this, set ACL groups in your config.py, or set DISABLE_ACL = True" + ) + return True + except: + pass + + return False diff --git a/will/backends/__init__.py b/will/backends/__init__.py new file mode 100644 index 00000000..066ded1d --- /dev/null +++ b/will/backends/__init__.py @@ -0,0 +1,6 @@ +from will.backends import analysis +from will.backends import execution +from will.backends import encryption +from will.backends import generation +from will.backends import pubsub +from will.backends import io_adapters diff --git a/will/backends/analysis/__init__.py b/will/backends/analysis/__init__.py new file mode 100644 index 00000000..1d36d3b3 --- /dev/null +++ b/will/backends/analysis/__init__.py @@ -0,0 +1,2 @@ +from .nothing import NoAnalysis +from .history import HistoryAnalysis diff --git a/will/backends/analysis/base.py b/will/backends/analysis/base.py new file mode 100644 index 00000000..46302d74 --- /dev/null +++ b/will/backends/analysis/base.py @@ -0,0 +1,53 @@ +import logging +import random +import signal +import time +import traceback +from multiprocessing.queues import Empty +from will import settings +from will.decorators import require_settings +from will.mixins import PubSubMixin, SleepMixin +from will.abstractions import Event + + +class AnalysisBackend(PubSubMixin, SleepMixin, object): + is_will_analysisbackend = True + + def __watch_pubsub(self): + while True: + try: + m = self.pubsub.get_message() + if m: + self.__analyze(m) + + except AttributeError: + pass + except (KeyboardInterrupt, SystemExit): + pass + self.sleep_for_event_loop() + + def __analyze(self, data): + try: + self.pubsub.publish( + "analysis.complete", + self.do_analyze(data), + reference_message=data + ) + except (KeyboardInterrupt, SystemExit): + pass + except: + logging.critical("Error completing analysis: \n%s" % traceback.format_exc()) + + def do_analyze(self, message): + # Take message, return a dict to add to its context. + raise NotImplemented + + def start(self, name, **kwargs): + signal.signal(signal.SIGINT, signal.SIG_IGN) + for k, v in kwargs.items(): + self.__dict__[k] = v + + self.name = name + self.bootstrap_pubsub() + self.subscribe("analysis.start") + self.__watch_pubsub() diff --git a/will/backends/analysis/history.py b/will/backends/analysis/history.py new file mode 100644 index 00000000..570ad296 --- /dev/null +++ b/will/backends/analysis/history.py @@ -0,0 +1,30 @@ +import requests + +from will import settings +from will.mixins import StorageMixin +from will.decorators import require_settings +from .base import AnalysisBackend + + +class HistoryAnalysis(AnalysisBackend, StorageMixin): + + def do_analyze(self, message): + # Load the last few messages, add it to the context under "history" + history = self.load("message_history", []) + if not history: + history = [] + max_history_context = getattr(settings, "HISTORY_CONTEXT_LENGTH", 20) + + if history: + context = { + "history": history[:max_history_context] + } + else: + context = { + "history": [], + } + + history.append(message) + self.save("message_history", history) + + return context diff --git a/will/backends/analysis/nothing.py b/will/backends/analysis/nothing.py new file mode 100644 index 00000000..4a3f5c8d --- /dev/null +++ b/will/backends/analysis/nothing.py @@ -0,0 +1,11 @@ +import requests + +from will import settings +from will.decorators import require_settings +from .base import AnalysisBackend + + +class NoAnalysis(AnalysisBackend): + + def do_analyze(self, message): + return {} diff --git a/will/backends/encryption/__init__.py b/will/backends/encryption/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/will/backends/encryption/aes.py b/will/backends/encryption/aes.py new file mode 100644 index 00000000..2cd31b11 --- /dev/null +++ b/will/backends/encryption/aes.py @@ -0,0 +1,65 @@ +import binascii +import base64 +import codecs +import dill as pickle +import hashlib +import logging +from Crypto.Cipher import AES +import random +import os +import traceback + +from will import settings +from will.backends.encryption.base import WillBaseEncryptionBackend + + +BS = 16 +key = hashlib.sha256(settings.SECRET_KEY.encode("utf-8")).digest() + + +def pad(s): + s = "%s%s" % (s.decode("utf-8"), ((BS - len(s) % BS) * "~")) + return s + + +def unpad(s): + while s.endswith(str.encode("~")): + s = s[:-1] + return s + + +class AESEncryption(WillBaseEncryptionBackend): + + @classmethod + def encrypt_to_b64(cls, raw): + try: + enc = binascii.b2a_base64(pickle.dumps(raw, -1)) + if settings.ENABLE_INTERNAL_ENCRYPTION: + iv = binascii.b2a_hex(os.urandom(8)) + cipher = AES.new(key, AES.MODE_CBC, iv) + enc = binascii.b2a_base64(cipher.encrypt(pad(enc))) + return "%s/%s" % (iv.decode("utf-8"), enc.decode("utf-8")) + else: + return enc + except: + logging.critical("Error preparing message for the wire: \n%s" % traceback.format_exc()) + return None + + @classmethod + def decrypt_from_b64(cls, raw_enc): + try: + if settings.ENABLE_INTERNAL_ENCRYPTION: + iv = raw_enc[:BS] + enc = raw_enc[BS+1:] + cipher = AES.new(key, AES.MODE_CBC, iv) + enc = unpad(cipher.decrypt(binascii.a2b_base64(enc))) + return pickle.loads(binascii.a2b_base64(enc)) + except (KeyboardInterrupt, SystemExit): + pass + except: + logging.warn("Error decrypting. Attempting unencrypted load to ease migration.") + return pickle.loads(binascii.a2b_base64(raw_enc)) + + +def bootstrap(settings): + return AESEncryption(settings) diff --git a/will/backends/encryption/base.py b/will/backends/encryption/base.py new file mode 100644 index 00000000..da4b0824 --- /dev/null +++ b/will/backends/encryption/base.py @@ -0,0 +1,12 @@ +class WillBaseEncryptionBackend(object): + + def __init__(self, *args, **kwargs): + pass + + @staticmethod + def encrypt_to_b64(raw): + raise NotImplemented + + @staticmethod + def decrypt_from_b64(enc): + raise NotImplemented diff --git a/will/backends/execution/__init__.py b/will/backends/execution/__init__.py new file mode 100644 index 00000000..99b2457c --- /dev/null +++ b/will/backends/execution/__init__.py @@ -0,0 +1,2 @@ +from .all import AllBackend +from .best_score import BestScoreBackend diff --git a/will/backends/execution/all.py b/will/backends/execution/all.py new file mode 100644 index 00000000..fbff1379 --- /dev/null +++ b/will/backends/execution/all.py @@ -0,0 +1,33 @@ + +import logging +import traceback +import requests +import warnings + +from will import settings +from will.decorators import require_settings +from .base import ExecutionBackend + + +class AllBackend(ExecutionBackend): + + def handle_execution(self, message): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + try: + had_one_reply = False + for m in message.generation_options: + self.execute(message, m) + had_one_reply = True + + if not had_one_reply: + self.no_response(message) + + return {} + except: + logging.critical( + "Error running %s. \n\n%s\nContinuing...\n" % ( + message.context.full_method_name, + traceback.format_exc() + ) + ) diff --git a/will/backends/execution/base.py b/will/backends/execution/base.py new file mode 100644 index 00000000..c075a115 --- /dev/null +++ b/will/backends/execution/base.py @@ -0,0 +1,111 @@ +import imp +import logging +import signal +import traceback +from will import settings +from will.decorators import require_settings +from will.acl import test_acl +from will.abstractions import Event +from multiprocessing import Process + + +class ExecutionBackend(object): + is_will_execution_backend = True + + def handle_execution(self, message, context): + raise NotImplemented + + def no_response(self, message): + self.bot.pubsub.publish( + "message.no_response", + message.data, + reference_message=message.data.original_incoming_event + ) + + def not_allowed(self, message, explanation): + + self.bot.pubsub.publish( + "message.outgoing.%s" % message.data.backend, + Event( + type="reply", + content=explanation, + source_message=message, + ), + reference_message=message.data.original_incoming_event + ) + + def execute(self, message, option): + if "acl" in option.context: + acl = option.context["acl"] + if type(acl) == type("test"): + acl = [acl] + + allowed = True + if len(acl) > 0: + allowed = test_acl(message, acl) + + if not allowed: + acl_list = "" + more_than_one_s = "" + if len(acl) > 1: + more_than_one_s = "s" + for i in range(0, len(acl)): + if i == 0: + acl_list = "%s" % acl[i] + elif i == len(acl) - 1: + acl_list = "%s or %s" % (acl_list, acl[i]) + else: + acl_list = "%s, %s" % (acl_list, acl[i]) + explanation = "Sorry, but I don't have you listed in the %s group%s, which is required to do what you asked." % (acl_list, more_than_one_s) + + self.not_allowed( + message, + explanation + ) + return + + if "say_content" in option.context: + # We're coming from a generation engine like a chatterbot, which doesn't *do* things. + self.bot.pubsub.publish( + "message.outgoing.%s" % message.data.backend, + Event( + type="reply", + content=option.context["say_content"], + source_message=message, + ), + reference_message=message.data.original_incoming_event + ) + else: + module = imp.load_source(option.context.plugin_info["parent_name"], option.context.plugin_info["parent_path"]) + cls = getattr(module, option.context.plugin_info["name"]) + + instantiated_module = cls(message=message) + method = getattr(instantiated_module, option.context.function_name) + + thread_args = [message, ] + option.context["args"] + + self.run_execute( + method, + *thread_args, + **option.context.search_matches + ) + + def run_execute(self, target, *args, **kwargs): + try: + t = Process( + target=target, + args=args, + kwargs=kwargs, + ) + self.bot.running_execution_threads.append(t) + t.start() + except (KeyboardInterrupt, SystemExit): + pass + except: + logging.critical("Error running %s: \n%s" % (target, traceback.format_exc())) + + def __init__(self, bot=None, *args, **kwargs): + self.bot = bot + if not bot: + raise Exception("Can't proceed without an instance of bot passed to the backend.") + super(ExecutionBackend, self).__init__(*args, **kwargs) diff --git a/will/backends/execution/best_score.py b/will/backends/execution/best_score.py new file mode 100644 index 00000000..4baec304 --- /dev/null +++ b/will/backends/execution/best_score.py @@ -0,0 +1,55 @@ +import imp +import logging +import traceback +import requests +import warnings + +from will import settings +from will.decorators import require_settings +from will.utils import Bunch +from .base import ExecutionBackend + + +class BestScoreBackend(ExecutionBackend): + + def _publish_fingerprint(self, option, message): + if "say_content" in option.context: + # TODO: Fix this to properly fingerprint + return option.context["say_content"] + else: + return "%s - %s" % (option.context.plugin_info["full_module_name"], option.context.full_method_name) + + def handle_execution(self, message): + published_list = [] + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + try: + had_one_reply = False + logging.info("message.generation_options") + logging.info(message.generation_options) + top_score = -1 + for m in message.generation_options: + logging.debug(m) + if m.score > top_score: + top_score = m.score + logging.debug("top_score") + logging.debug(top_score) + for m in message.generation_options: + if m.score >= top_score: + s = self._publish_fingerprint(m, message) + if not s in published_list: + published_list.append(s) + self.execute(message, m) + had_one_reply = True + + if not had_one_reply: + self.no_response(message) + + return {} + except: + logging.critical( + "Error running %s. \n\n%s\nContinuing...\n" % ( + message, + traceback.format_exc() + ) + ) diff --git a/will/backends/generation/__init__.py b/will/backends/generation/__init__.py new file mode 100644 index 00000000..1f505008 --- /dev/null +++ b/will/backends/generation/__init__.py @@ -0,0 +1,3 @@ +from .strict_regex import RegexBackend +from .fuzzy_best_match import FuzzyBestMatch +from .fuzzy_all_matches import FuzzyAllMatchesBackend diff --git a/will/backends/generation/base.py b/will/backends/generation/base.py new file mode 100644 index 00000000..59b5d8a5 --- /dev/null +++ b/will/backends/generation/base.py @@ -0,0 +1,72 @@ +import logging +import random +import time +import traceback +import dill as pickle +from will import settings +from will.decorators import require_settings +from will.mixins import PubSubMixin, SleepMixin +from will.abstractions import Event +from will.utils import Bunch + + +class GenerationBackend(PubSubMixin, SleepMixin, object): + is_will_generationbackend = True + + def __watch_pubsub(self): + while True: + try: + m = self.pubsub.get_message() + if m: + self.__generate(m.data) + except (KeyboardInterrupt, SystemExit): + pass + self.sleep_for_event_loop() + + def __generate(self, message): + ret = self.do_generate(message) + try: + self.pubsub.publish("generation.complete", ret, reference_message=message) + except (KeyboardInterrupt, SystemExit): + pass + except: + logging.critical("Error publishing generation.complete: \n%s" % traceback.format_exc()) + + def do_generate(self, message): + # Take message, return a list of possible responses/matches + raise NotImplemented + + def start(self, name, **kwargs): + for k, v in kwargs.items(): + self.__dict__[k] = v + + self.name = name + self.bootstrap_pubsub() + self.subscribe("generation.start") + self.__watch_pubsub() + + +OPTION_REQUIRED_FIELDS = [ + "backend", + "context", + "score", +] + + +class GeneratedOption(object): + + def __init__(self, *args, **kwargs): + for o in OPTION_REQUIRED_FIELDS: + if o not in kwargs: + raise Exception("Missing '%s' argument to the generator backend." % o) + + for k, v in kwargs.items(): + self.__dict__[k] = v + + return super(GeneratedOption, self).__init__() + + def __unicode__(self): + return "%s - %s" % (self.score, self.context) + + def __str__(self): + return "%s - %s" % (self.score, self.context) diff --git a/will/backends/generation/fuzzy_all_matches.py b/will/backends/generation/fuzzy_all_matches.py new file mode 100644 index 00000000..b07faf36 --- /dev/null +++ b/will/backends/generation/fuzzy_all_matches.py @@ -0,0 +1,106 @@ +import logging +import regex +from fuzzywuzzy import fuzz +from fuzzywuzzy import process as fuzz_process +from will import settings +from will.decorators import require_settings +from will.utils import Bunch +from .base import GenerationBackend, GeneratedOption + + +class FuzzyAllMatchesBackend(GenerationBackend): + + def _generate_compiled_regex(self, method_meta): + if not hasattr(self, "cached_regex"): + self.cached_regex = {} + + method_path = method_meta["plugin_info"]["parent_path"] + if not method_path in self.cached_regex: + + regex_string = method_meta["regex_pattern"] + if "case_sensitive" in method_meta and not method_meta["case_sensitive"]: + regex_string = "(?i)%s" % regex_string + + if method_meta["multiline"]: + try: + self.cached_regex[method_path] = regex.compile("%s{e<=%s}" % ( + regex_string, + settings.FUZZY_REGEX_ALLOWABLE_ERRORS + ), regex.MULTILINE | regex.DOTALL | regex.ENHANCEMATCH) + except: + self.cached_regex[method_path] = regex.compile("%s{e<=%s}" % ( + regex.escape(regex_string), + settings.FUZZY_REGEX_ALLOWABLE_ERRORS + ), regex.MULTILINE | regex.DOTALL | regex.ENHANCEMATCH) + else: + try: + self.cached_regex[method_path] = regex.compile("%s{e<=%s}" % ( + regex_string, + settings.FUZZY_REGEX_ALLOWABLE_ERRORS + ), regex.ENHANCEMATCH) + except: + self.cached_regex[method_path] = regex.compile("%s{e<=%s}" % ( + regex.escape(regex_string), + settings.FUZZY_REGEX_ALLOWABLE_ERRORS + ), regex.ENHANCEMATCH) + + return self.cached_regex[method_path] + + def do_generate(self, event): + exclude_list = ["fn", ] + matches = [] + + message = event.data + + # TODO: add token_sort_ratio + if message.content: + if not hasattr(self, "match_choices"): + self.match_choices = [] + self.match_methods = {} + for name, l in self.bot.message_listeners.items(): + if not l["regex_pattern"] in self.match_methods: + self.match_methods[l["regex_pattern"]] = l + self.match_choices.append(l["regex_pattern"]) + + search_matches = fuzz_process.extract(message.content, self.match_choices) + + for match_str, confidence in search_matches: + logging.debug(" Potential (%s) - %s" % (confidence, match_str)) + l = self.match_methods[match_str] + if ( + # The search regex matches and + # regex_matches + + # We're confident enough + (confidence >= settings.FUZZY_MINIMUM_MATCH_CONFIDENCE) + + # It's not from me, or this search includes me, and + and ( + message.will_said_it is False or + ("include_me" in l and l["include_me"]) + ) + + # I'm mentioned, or this is an overheard, or we're in a 1-1 + and ( + message.is_private_chat or + ("direct_mentions_only" not in l or not l["direct_mentions_only"]) or + message.is_direct + ) + ): + logging.info(" Match (%s) - %s" % (confidence, match_str)) + fuzzy_regex = self._generate_compiled_regex(l) + + regex_matches = fuzzy_regex.search(message.content) + context = Bunch() + for k, v in l.items(): + if k not in exclude_list: + context[k] = v + if regex_matches and hasattr(regex_matches, "groupdict"): + context.search_matches = regex_matches.groupdict() + else: + context.search_matches = {} + + o = GeneratedOption(context=context, backend="regex", score=confidence) + matches.append(o) + + return matches diff --git a/will/backends/generation/fuzzy_best_match.py b/will/backends/generation/fuzzy_best_match.py new file mode 100644 index 00000000..dba595c9 --- /dev/null +++ b/will/backends/generation/fuzzy_best_match.py @@ -0,0 +1,101 @@ +from fuzzywuzzy import fuzz +from fuzzywuzzy import process as fuzz_process +import regex +from will import settings +from will.decorators import require_settings +from will.utils import Bunch +from .base import GenerationBackend, GeneratedOption + + +class FuzzyBestMatch(GenerationBackend): + + def _generate_compiled_regex(self, method_meta): + if not hasattr(self, "cached_regex"): + self.cached_regex = {} + + method_path = method_meta["plugin_info"]["parent_path"] + if not method_path in self.cached_regex: + + regex_string = method_meta["regex_pattern"] + if "case_sensitive" in method_meta and not method_meta["case_sensitive"]: + regex_string = "(?i)%s" % regex_string + + if method_meta["multiline"]: + try: + self.cached_regex[method_path] = regex.compile("%s{e<=%s}" % ( + regex_string, + settings.FUZZY_REGEX_ALLOWABLE_ERRORS + ), regex.MULTILINE | regex.DOTALL | regex.ENHANCEMATCH) + except: + self.cached_regex[method_path] = regex.compile("%s{e<=%s}" % ( + regex.escape(regex_string), + settings.FUZZY_REGEX_ALLOWABLE_ERRORS + ), regex.MULTILINE | regex.DOTALL | regex.ENHANCEMATCH) + else: + try: + self.cached_regex[method_path] = regex.compile("%s{e<=%s}" % ( + regex_string, + settings.FUZZY_REGEX_ALLOWABLE_ERRORS + ), regex.ENHANCEMATCH) + except: + self.cached_regex[method_path] = regex.compile("%s{e<=%s}" % ( + regex.escape(regex_string), + settings.FUZZY_REGEX_ALLOWABLE_ERRORS + ), regex.ENHANCEMATCH) + + return self.cached_regex[method_path] + + def do_generate(self, event): + exclude_list = ["fn", ] + matches = [] + + message = event.data + + # TODO: add token_sort_ratio + + if not hasattr(self, "match_choices"): + self.match_choices = [] + self.match_methods = {} + if message.content: + for name, l in self.bot.message_listeners.items(): + if not l["regex_pattern"] in self.match_methods: + self.match_methods[l["regex_pattern"]] = l + self.match_choices.append(l["regex_pattern"]) + + match_str, confidence = fuzz_process.extractOne(message.content, self.match_choices) + l = self.match_methods[match_str] + if confidence >= settings.FUZZY_MINIMUM_MATCH_CONFIDENCE: + regex_matches = l["regex"].search(message.content) + if ( + # The search regex matches and + # regex_matches + + # It's not from me, or this search includes me, and + ( + message.will_said_it is False or + ("include_me" in l and l["include_me"]) + ) + + # I'm mentioned, or this is an overheard, or we're in a 1-1 + and ( + message.is_private_chat or + ("direct_mentions_only" not in l or not l["direct_mentions_only"]) or + message.is_direct + ) + ): + fuzzy_regex = self._generate_compiled_regex(l) + + regex_matches = fuzzy_regex.search(message.content) + context = Bunch() + for k, v in l.items(): + if k not in exclude_list: + context[k] = v + if regex_matches and hasattr(regex_matches, "groupdict"): + context.search_matches = regex_matches.groupdict() + else: + context.search_matches = {} + + o = GeneratedOption(context=context, backend="regex", score=confidence) + matches.append(o) + + return matches diff --git a/will/backends/generation/strict_regex.py b/will/backends/generation/strict_regex.py new file mode 100644 index 00000000..c8d8ed19 --- /dev/null +++ b/will/backends/generation/strict_regex.py @@ -0,0 +1,43 @@ +import re +from will import settings +from will.decorators import require_settings +from will.utils import Bunch +from .base import GenerationBackend, GeneratedOption + + +class RegexBackend(GenerationBackend): + + def do_generate(self, event): + exclude_list = ["fn", ] + matches = [] + + message = event.data + for name, l in self.bot.message_listeners.items(): + search_matches = l["regex"].search(message.content) + if ( + # The search regex matches and + search_matches + + # It's not from me, or this search includes me, and + and ( + message.will_said_it is False or + ("include_me" in l and l["include_me"]) + ) + + # I'm mentioned, or this is an overheard, or we're in a 1-1 + and ( + message.is_private_chat or + ("direct_mentions_only" not in l or not l["direct_mentions_only"]) or + message.is_direct + ) + ): + context = Bunch() + for k, v in l.items(): + if k not in exclude_list: + context[k] = v + context.search_matches = search_matches.groupdict() + + o = GeneratedOption(context=context, backend="regex", score=100) + matches.append(o) + + return matches diff --git a/will/backends/io_adapters/__init__.py b/will/backends/io_adapters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/will/backends/io_adapters/base.py b/will/backends/io_adapters/base.py new file mode 100644 index 00000000..5267ee27 --- /dev/null +++ b/will/backends/io_adapters/base.py @@ -0,0 +1,127 @@ +# -- coding: utf-8 - +from clint.textui import indent, puts +import datetime +import hashlib +import logging +from pytz import timezone as pytz_timezone +import signal +import time +import traceback + +from will import settings +from will.utils import Bunch, show_valid, error, warn +from will.mixins import PubSubMixin, SleepMixin, SettingsMixin +from will.abstractions import Message, Event, Person +from multiprocessing import Process + + +class IOBackend(PubSubMixin, SleepMixin, SettingsMixin, object): + is_will_iobackend = True + required_settings = [] + + def bootstrap(self): + raise NotImplemented("""A .bootstrap() method was not provided. + + Bootstrap must provide a way to to have: + a) self.normalize_incoming_event fired, or incoming events put into self.incoming_queue + b) any necessary threads running for a) + c) self.handle (string) defined + d) self.me (Person) defined, with Will's info + e) self.people (dict of People) defined, with everyone in an organization/backend + f) self.channels (dict of Channels) defined, with all available channels/rooms. + Note that Channel asks for members, a list of People. + g) A way for self.handle, self.me, self.people, and self.channels to be kept accurate, + with a maximum lag of 60 seconds. + """) + + def normalize_incoming_event(self, event): + # Takes a raw event, converts it into a Message, and returns the normalized Message. + raise NotImplemented + + def handle_incoming_event(self, event): + try: + m = self.normalize_incoming_event(event) + if m: + self.pubsub.publish("message.incoming", m, reference_message=m) + except: + logging.critical("Error handling incoming event %s: \n%s" % ( + event, + traceback.format_exc(), + )) + + def handle_outgoing_event(self, event): + raise NotImplemented + + def terminate(self): + pass + + def __publish_incoming_message(self, message): + return self.pubsub.publish("message.incoming", message, reference_message=message) + + def __start_event_listeners(self): + signal.signal(signal.SIGINT, signal.SIG_IGN) + running = True + while running: + try: + pubsub_event = self.pubsub.get_message() + if pubsub_event: + if pubsub_event.type == "message.incoming": + self.handle_incoming_event(pubsub_event) + elif pubsub_event.type == "message.outgoing.%s" % self.name: + self.handle_outgoing_event(pubsub_event.data) + elif pubsub_event.type == "message.incoming.stdin": + self.handle_incoming_event(pubsub_event) + elif pubsub_event.type == "message.no_response.%s" % self.name: + self.handle_outgoing_event(pubsub_event) + elif pubsub_event.type == "system.terminate": + self.__handle_terminate() + running = False + except (KeyboardInterrupt, SystemExit): + pass + self.sleep_for_event_loop() + + def __handle_terminate(self): + if hasattr(self, "__event_listener_thread"): + logging.debug("__event_listener_thread") + try: + self.__event_listener_thread.terminate() + while self.__event_listener_thread.is_alive(): + self.sleep_for_event_loop() + except (KeyboardInterrupt, SystemExit): + pass + if hasattr(self, "terminate"): + try: + self.terminate() + except (KeyboardInterrupt, SystemExit): + pass + + def _start(self, name): + try: + self.name = name + self.bootstrap_pubsub() + self.pubsub.subscribe([ + "message.incoming", + "message.outgoing.%s" % self.name, + "message.no_response.%s" % self.name, + "system.terminate", + ]) + if hasattr(self, "stdin_process") and self.stdin_process: + self.pubsub.subscribe(["message.incoming.stdin", ]) + + self.__event_listener_thread = Process( + target=self.__start_event_listeners, + ) + self.__event_listener_thread.start() + + self.bootstrap() + except (KeyboardInterrupt, SystemExit): + pass + except: + logging.critical("Error starting io adapter %s: \n%s" % ( + self.name, + traceback.format_exc(), + )) + + +class StdInOutIOBackend(IOBackend): + stdin_process = True diff --git a/will/backends/io_adapters/hipchat.py b/will/backends/io_adapters/hipchat.py new file mode 100644 index 00000000..fd18852a --- /dev/null +++ b/will/backends/io_adapters/hipchat.py @@ -0,0 +1,800 @@ +from datetime import datetime +import json +import logging +from multiprocessing.queues import Empty +from multiprocessing import Process, Queue +import random +import re +import requests +import pickle +import sys +import time +import threading +import traceback + +from sleekxmpp import ClientXMPP +from sleekxmpp.exceptions import IqError, IqTimeout + +from .base import IOBackend +from will import settings +from will.utils import is_admin +from will.acl import is_acl_allowed +from will.abstractions import Event, Message, Person, Channel +from will.utils import Bunch, UNSURE_REPLIES, clean_for_pickling +from will.mixins import StorageMixin, PubSubMixin + +ROOM_NOTIFICATION_URL = "https://%(server)s/v2/room/%(room_id)s/notification?auth_token=%(token)s" +ROOM_TOPIC_URL = "https://%(server)s/v2/room/%(room_id)s/topic?auth_token=%(token)s" +ROOM_URL = "https://%(server)s/v2/room/%(room_id)s/?auth_token=%(token)s" +SET_TOPIC_URL = "https://%(server)s/v2/room/%(room_id)s/topic?auth_token=%(token)s" +PRIVATE_MESSAGE_URL = "https://%(server)s/v2/user/%(user_id)s/message?auth_token=%(token)s" +USER_DETAILS_URL = "https://%(server)s/v2/user/%(user_id)s?auth_token=%(token)s" +ALL_USERS_URL = ("https://%(server)s/v2/user?auth_token=%(token)s&start-index" + "=%(start_index)s&max-results=%(max_results)s") +ALL_ROOMS_URL = ("https://%(server)s/v2/room?auth_token=%(token)s&start-index" + "=%(start_index)s&max-results=%(max_results)s&expand=items") + +# From RoomsMixins +V1_TOKEN_URL = "https://%(server)s/v1/rooms/list?auth_token=%(token)s" +V2_TOKEN_URL = "https://%(server)s/v2/room?auth_token=%(token)s&expand=items" + + +class HipChatRosterMixin(object): + @property + def people(self): + if not hasattr(self, "_people"): + self._people = self.load('will_hipchat_people', {}) + return self._people + + @property + def internal_roster(self): + logging.warn( + "mixin.internal_roster has been deprecated. Please use mixin.people instead. " + "internal_roster will be removed at the end of 2017" + ) + return self.people + + def get_user_by_full_name(self, name): + for jid, info in self.people.items(): + if info["name"] == name: + return info + + return None + + def get_user_by_nick(self, nick): + for jid, info in self.people.items(): + if info["nick"] == nick: + return info + return None + + def get_user_by_jid(self, jid): + if jid in self.people: + return self.people[jid] + + return None + + def get_user_from_message(self, message): + if message["type"] == "groupchat": + if "xmpp_jid" in message: + user = self.get_user_by_jid(message["xmpp_jid"]) + if user: + return user + elif "from" in message: + full_name = message["from"].split("/")[1] + user = self.get_user_by_full_name(full_name) + if user: + return user + + if "mucnick" in message: + return self.get_user_by_full_name(message["mucnick"]) + + elif message['type'] in ('chat', 'normal'): + jid = ("%s" % message["from"]).split("@")[0].split("_")[1] + return self.get_user_by_jid(jid) + else: + return None + + def message_is_from_admin(self, message): + nick = self.get_user_from_message(message)['nick'] + return is_admin(nick) + + def message_is_allowed(self, message, acl): + nick = self.get_user_from_message(message)['nick'] + return is_acl_allowed(nick, acl) + + def get_user_by_hipchat_id(self, id): + for jid, info in self.people.items(): + if info["hipchat_id"] == id: + return info + return None + + +class HipChatRoom(Bunch): + + @property + def id(self): + if 'room_id' in self: + # Using API v1 + return self['room_id'] + elif 'id' in self: + # Using API v2 + return self['id'] + else: + raise TypeError('Room ID not found') + + @property + def history(self): + payload = {"auth_token": settings.HIPCHAT_V2_TOKEN} + response = requests.get("https://{1}/v2/room/{0}/history".format(str(self.id), + settings.HIPCHAT_SERVER), + params=payload, **settings.REQUESTS_OPTIONS) + data = json.loads(response.text)['items'] + for item in data: + item['date'] = datetime.strptime(item['date'][:-13], "%Y-%m-%dT%H:%M:%S") + return data + + @property + def participants(self): + payload = {"auth_token": settings.HIPCHAT_V2_TOKEN} + response = requests.get( + "https://{1}/v2/room/{0}/participant".format( + str(self.id), + settings.HIPCHAT_SERVER + ), + params=payload, + **settings.REQUESTS_OPTIONS + ).json() + data = response['items'] + while 'next' in response['links']: + response = requests.get(response['links']['next'], + params=payload, **settings.REQUESTS_OPTIONS).json() + data.extend(response['items']) + return data + + +class HipChatRoomMixin(object): + def update_available_rooms(self, q=None): + self._available_rooms = {} + # Use v1 token to grab a full room list if we can (good to avoid rate limiting) + if hasattr(settings, "V1_TOKEN"): + url = V1_TOKEN_URL % {"server": settings.HIPCHAT_SERVER, + "token": settings.HIPCHAT_V1_TOKEN} + r = requests.get(url, **settings.REQUESTS_OPTIONS) + if r.status_code == requests.codes.unauthorized: + raise Exception("V1_TOKEN authentication failed with HipChat") + for room in r.json()["rooms"]: + # Some integrations expect a particular name for the ID field. + # Better to use room.id. + room["id"] = room["room_id"] + self._available_rooms[room["name"]] = HipChatRoom(**room) + # Otherwise, grab 'em one-by-one via the v2 api. + else: + params = {} + params['start-index'] = 0 + max_results = params['max-results'] = 1000 + url = V2_TOKEN_URL % {"server": settings.HIPCHAT_SERVER, + "token": settings.HIPCHAT_V2_TOKEN} + while True: + resp = requests.get(url, params=params, + **settings.REQUESTS_OPTIONS) + if resp.status_code == requests.codes.unauthorized: + raise Exception("V2_TOKEN authentication failed with HipChat") + rooms = resp.json() + + for room in rooms["items"]: + # Some integrations expect a particular name for the ID field. + # Better to use room.id + room["room_id"] = room["id"] + self._available_rooms[room["name"]] = HipChatRoom(**room) + + logging.info('Got %d rooms', len(rooms['items'])) + if len(rooms['items']) == max_results: + params['start-index'] += max_results + else: + break + + self.save("hipchat_rooms", self._available_rooms) + if q: + q.put(self._available_rooms) + + @property + def available_rooms(self): + if not hasattr(self, "_available_rooms"): + self._available_rooms = self.load('hipchat_rooms', None) + if not self._available_rooms: + self.update_available_rooms() + + return self._available_rooms + + def get_room_by_jid(self, jid): + for room in self.available_rooms.values(): + if "xmpp_jid" in room and room["xmpp_jid"] == jid: + return room + return None + + def get_room_from_message(self, message): + return self.get_room_from_name_or_id(message.data.channel.name) + + def get_room_from_name_or_id(self, name_or_id): + for name, room in self.available_rooms.items(): + if name_or_id.lower() == name.lower(): + return room + if "xmpp_jid" in room and name_or_id == room["xmpp_jid"]: + return room + if "room_id" in room and name_or_id == room["room_id"]: + return room + return None + + +class HipChatXMPPClient(ClientXMPP, HipChatRosterMixin, HipChatRoomMixin, StorageMixin, PubSubMixin): + + def start_xmpp_client(self, xmpp_bridge_queue=None, backend_name=""): + logger = logging.getLogger(__name__) + if not xmpp_bridge_queue: + logger.error("Missing required bridge queue") + + self.xmpp_bridge_queue = xmpp_bridge_queue + self.backend_name = backend_name + + ClientXMPP.__init__(self, "%s/bot" % settings.HIPCHAT_USERNAME, settings.HIPCHAT_PASSWORD) + + if settings.USE_PROXY: + self.use_proxy = True + self.proxy_config = { + 'host': settings.PROXY_HOSTNAME, + 'port': settings.PROXY_PORT, + 'username': settings.PROXY_USERNAME, + 'password': settings.PROXY_PASSWORD, + } + + self.rooms = [] + self.default_room = settings.HIPCHAT_DEFAULT_ROOM + + my_user_url = "https://%(server)s/v2/user/%(user_id)s?auth_token=%(token)s" % { + "user_id": settings.HIPCHAT_USERNAME.split("@")[0].split("_")[1], + "server": settings.HIPCHAT_SERVER, + "token": settings.HIPCHAT_V2_TOKEN, + } + + r = requests.get(my_user_url, **settings.REQUESTS_OPTIONS) + resp = r.json() + if "email" in resp: + settings.HIPCHAT_EMAIL = resp["email"] + settings.HIPCHAT_HANDLE = resp["mention_name"] + settings.HIPCHAT_NAME = resp["name"] + else: + raise EnvironmentError( + "\n\nError getting user info from Hipchat. This is usually a problem with the\n" + "username or V2 token, but here's what I heard back from them: \n\n %s\n\n" % resp + ) + + self.available_rooms + if hasattr(settings, "HIPCHAT_ROOMS") and settings.HIPCHAT_ROOMS: + for r in settings.HIPCHAT_ROOMS: + if r != "": + if not hasattr(self, "default_room"): + self.default_room = r + + try: + self.rooms.append(self.available_rooms[r]) + except KeyError: + logger.error( + u'"{0}" is not an available room, ask' + ' "@{1} what are the rooms?" for the full list.' + .format(r, settings.HIPCHAT_HANDLE)) + else: + for name, r in self.available_rooms.items(): + if not hasattr(self, "default_room"): + self.default_room = r + self.rooms.append(r) + + self.nick = settings.HIPCHAT_HANDLE + self.handle = settings.HIPCHAT_HANDLE + self.mention_handle = "@%s" % settings.HIPCHAT_HANDLE + + self.whitespace_keepalive = True + self.whitespace_keepalive_interval = 30 + + if settings.ALLOW_INSECURE_HIPCHAT_SERVER is True: + self.add_event_handler('ssl_invalid_cert', lambda cert: True) + + self.add_event_handler("roster_update", self.join_rooms) + self.add_event_handler("session_start", self.session_start) + self.add_event_handler("message", self.message_recieved) + self.add_event_handler("groupchat_message", self.room_message) + self.add_event_handler("groupchat_invite", self.room_invite) + self.add_event_handler("error", self.handle_errors) + self.add_event_handler("presence_error", self.handle_errors) + + self.register_plugin('xep_0045') # MUC + + def session_start(self, event): + self.send_presence() + try: + self.get_roster() + except IqError as err: + logging.error('There was an error getting the roster') + logging.error(err.iq['error']['condition']) + self.disconnect() + except IqTimeout: + logging.error('Server is taking too long to respond. Disconnecting.') + self.disconnect() + + def join_rooms(self, event): + for r in self.rooms: + if "xmpp_jid" in r: + self.plugin['xep_0045'].joinMUC(r["xmpp_jid"], settings.HIPCHAT_NAME, wait=True) + + def handle_errors(self, event): + print("got error event") + print(event) + + def room_invite(self, event): + logging.info("Invite recieved for %s" % event) + for r in self.rooms: + if "xmpp_jid" in r: + self.plugin['xep_0045'].joinMUC(r["xmpp_jid"], settings.HIPCHAT_NAME, wait=True) + + def update_will_roster_and_rooms(self): + people = self.load('will_hipchat_people', {}) + + # Loop through the connected rooms (self.roster comes from ClientXMPP) + for roster_id in self.roster: + + cur_roster = self.roster[roster_id] + # Loop through the users in a given room + for user_id in cur_roster: + user_data = cur_roster[user_id] + if user_data["name"] != "": + # If we don't have this user in the people, add them. + if not user_id in people: + people[user_id] = Person() + + hipchat_id = user_id.split("@")[0].split("_")[1] + # Update their info + people[user_id].update({ + "name": user_data["name"], + "jid": user_id, + "hipchat_id": hipchat_id, + }) + + # If we don't have a nick yet, pull it and mention_name off the master user list. + if not hasattr(people[user_id], "nick") and hipchat_id in self.people: + user_data = self.get_user_list[hipchat_id] + people[user_id].nick = user_data["mention_name"] + people[user_id].mention_name = user_data["mention_name"] + + # If it's me, save that info! + if people[user_id].get("name", "") == self.nick: + self.me = people[user_id] + + self.save("will_hipchat_people", people) + + self.update_available_rooms() + + def room_message(self, msg): + self._send_to_backend(msg) + + def message_recieved(self, msg): + if msg['type'] in ('chat', 'normal'): + self._send_to_backend(msg) + + def real_sender_jid(self, msg): + # There's a bug in sleekXMPP where it doesn't set the "from_jid" properly. + # Thus, this hideous hack. + msg_str = "%s" % msg + start = 'from_jid="' + start_pos = msg_str.find(start) + if start_pos != -1: + cut_start = start_pos + len(start) + return msg_str[cut_start:msg_str.find('"', cut_start)] + + return msg["from"] + + def _send_to_backend(self, msg): + stripped_msg = Bunch() + # TODO: Find a faster way to do this - this is crazy. + for k, v in msg.__dict__.items(): + try: + pickle.dumps(v) + stripped_msg[k] = v + except: + pass + for k in msg.xml.keys(): + try: + # print(k) + # print(msg.xml.get(k)) + pickle.dumps(msg.xml.get(k)) + stripped_msg[k] = msg.xml.get(k) + except: + # print("failed to parse %s" % k) + pass + + stripped_msg.xmpp_jid = msg.getMucroom() + stripped_msg.body = msg["body"] + self.xmpp_bridge_queue.put(stripped_msg) + + +class HipChatBackend(IOBackend, HipChatRosterMixin, HipChatRoomMixin, StorageMixin): + friendly_name = "HipChat" + internal_name = "will.backends.io_adapters.hipchat" + required_settings = [ + { + "name": "HIPCHAT_USERNAME", + "obtain_at": """1. Go to hipchat, and create a new user for will. +2. Log into will, and go to Account settings>XMPP/Jabber Info. +3. On that page, the 'Jabber ID' is the value you want to use.""", + }, + { + "name": "HIPCHAT_PASSWORD", + "obtain_at": ( + "1. Go to hipchat, and create a new user for will. " + "Note that password - this is the value you want. " + "It's used for signing in via XMPP." + ), + }, + { + "name": "HIPCHAT_V2_TOKEN", + "obtain_at": """1. Log into hipchat using will's user. +2. Go to https://your-org.hipchat.com/account/api +3. Create a token. +4. Copy the value - this is the HIPCHAT_V2_TOKEN.""", + } + ] + + def send_direct_message(self, user_id, message_body, html=False, card=None, notify=False, **kwargs): + if kwargs: + logging.warn("Unknown keyword args for send_direct_message: %s" % kwargs) + + format = "text" + if html: + format = "html" + + try: + # https://www.hipchat.com/docs/apiv2/method/private_message_user + url = PRIVATE_MESSAGE_URL % {"server": settings.HIPCHAT_SERVER, + "user_id": user_id, + "token": settings.HIPCHAT_V2_TOKEN} + data = { + "message": message_body, + "message_format": format, + "notify": notify, + "card": card, + } + headers = {'Content-type': 'application/json', 'Accept': 'text/plain'} + r = requests.post(url, headers=headers, data=json.dumps(data), **settings.REQUESTS_OPTIONS) + r.raise_for_status() + except: + logging.critical("Error in send_direct_message: \n%s" % traceback.format_exc()) + + def send_room_message(self, room_id, message_body, html=False, color="green", notify=False, card=None, **kwargs): + if kwargs: + logging.warn("Unknown keyword args for send_room_message: %s" % kwargs) + + format = "text" + if html: + format = "html" + + try: + # https://www.hipchat.com/docs/apiv2/method/send_room_notification + url = ROOM_NOTIFICATION_URL % {"server": settings.HIPCHAT_SERVER, + "room_id": room_id, + "token": settings.HIPCHAT_V2_TOKEN} + data = { + "message": message_body, + "message_format": format, + "color": color, + "notify": notify, + "card": card, + } + headers = {'Content-type': 'application/json', 'Accept': 'text/plain'} + r = requests.post(url, headers=headers, data=json.dumps(data), **settings.REQUESTS_OPTIONS) + r.raise_for_status() + + except: + logging.critical("Error in send_room_message: \n%s" % traceback.format_exc()) + + def set_room_topic(self, room_id, topic): + try: + # https://www.hipchat.com/docs/apiv2/method/send_room_notification + url = ROOM_TOPIC_URL % {"server": settings.HIPCHAT_SERVER, + "room_id": room_id, + "token": settings.HIPCHAT_V2_TOKEN} + data = { + "topic": topic, + } + headers = {'Content-type': 'application/json', 'Accept': 'text/plain'} + requests.put(url, headers=headers, data=json.dumps(data), **settings.REQUESTS_OPTIONS) + except: + logging.critical("Error in set_room_topic: \n%s" % traceback.format_exc()) + + def get_room_from_message(self, event): + kwargs = {} + if hasattr(event, "kwargs"): + kwargs.update(event.kwargs) + if hasattr(event, "source_message") and event.source_message: + send_source = event.source_message + if hasattr(event.source_message, "data"): + send_source = event.source_message.data + + if send_source.is_private_chat: + # Private, 1-1 chats. + return False + else: + # We're in a public room + return send_source.channel.id + else: + # Came from webhook/etc + if "room" in kwargs: + return kwargs["room"], + else: + return self.get_room_from_name_or_id(settings.HIPCHAT_DEFAULT_ROOM)["room_id"] + return False + + def get_hipchat_user(self, user_id, q=None): + url = USER_DETAILS_URL % {"server": settings.HIPCHAT_SERVER, + "user_id": user_id, + "token": settings.HIPCHAT_V2_TOKEN} + r = requests.get(url, **settings.REQUESTS_OPTIONS) + if q: + q.put(r.json()) + else: + return r.json() + + @property + def people(self): + if not hasattr(self, "_people"): + full_roster = {} + + # Grab the first roster page, and populate full_roster + url = ALL_USERS_URL % {"server": settings.HIPCHAT_SERVER, + "token": settings.HIPCHAT_V2_TOKEN, + "start_index": 0, + "max_results": 1000} + r = requests.get(url, **settings.REQUESTS_OPTIONS) + for user in r.json()['items']: + full_roster["%s" % (user['id'],)] = Person( + id=user["id"], + handle=user["mention_name"], + mention_handle="@%s" % user["mention_name"], + source=clean_for_pickling(user), + name=user["name"], + ) + # Keep going through the next pages until we're out of pages. + while 'next' in r.json()['links']: + url = "%s&auth_token=%s" % (r.json()['links']['next'], settings.HIPCHAT_V2_TOKEN) + r = requests.get(url, **settings.REQUESTS_OPTIONS) + + for user in r.json()['items']: + full_roster["%s" % (user['id'],)] = Person( + id=user["id"], + handle=user["mention_name"], + mention_handle="@%s" % user["mention_name"], + source=clean_for_pickling(user), + name=user["name"], + ) + + self._people = full_roster + for k, u in full_roster.items(): + if u.handle == settings.HIPCHAT_HANDLE: + self.me = u + return self._people + + @property + def channels(self): + if not hasattr(self, "_channels"): + all_rooms = {} + + # Grab the first roster page, and populate all_rooms + url = ALL_ROOMS_URL % {"server": settings.HIPCHAT_SERVER, + "token": settings.HIPCHAT_V2_TOKEN, + "start_index": 0, + "max_results": 1000} + r = requests.get(url, **settings.REQUESTS_OPTIONS) + for room in r.json()['items']: + # print(room) + all_rooms["%s" % (room['xmpp_jid'],)] = Channel( + id=room["id"], + name=room["name"], + source=clean_for_pickling(room), + members={}, + ) + + # Keep going through the next pages until we're out of pages. + while 'next' in r.json()['links']: + url = "%s&auth_token=%s" % (r.json()['links']['next'], settings.HIPCHAT_V2_TOKEN) + r = requests.get(url, **settings.REQUESTS_OPTIONS) + + for room in r.json()['items']: + all_rooms["%s" % (room['xmpp_jid'],)] = Channel( + id=room["id"], + name=room["name"], + source=clean_for_pickling(room), + members={} + ) + + self._channels = all_rooms + return self._channels + + def normalize_incoming_event(self, event): + logging.debug("hipchat: normalize_incoming_event - %s" % event) + if event["type"] in ("chat", "normal", "groupchat") and ("from_jid" in event or "from" in event): + + sender = self.get_user_from_message(event) + interpolated_handle = "@%s" % self.me.handle + will_is_mentioned = False + will_said_it = False + channel = None + if "xmpp_jid" in event and event["xmpp_jid"]: + channel = clean_for_pickling(self.channels[event["xmpp_jid"]]) + is_private_chat = False + else: + if event["type"] in ("chat", "normal"): + is_private_chat = True + + is_direct = False + if is_private_chat or event["body"].startswith(interpolated_handle): + is_direct = True + + if event["body"].startswith(interpolated_handle): + event["body"] = event["body"][len(interpolated_handle):].strip() + + if interpolated_handle in event["body"]: + will_is_mentioned = True + + if sender and self.me and sender.id == self.me.id: + will_said_it = True + + m = Message( + content=event["body"], + is_direct=is_direct, + is_private_chat=is_private_chat, + is_group_chat=not is_private_chat, + backend=self.internal_name, + sender=sender, + channel=channel, + will_is_mentioned=will_is_mentioned, + will_said_it=will_said_it, + backend_supports_acl=True, + original_incoming_event=clean_for_pickling(event), + ) + # print("normalized:") + # print(m.__dict__) + return m + + else: + # print("Unknown event type") + # print(event) + return None + + def handle_outgoing_event(self, event): + kwargs = {} + if hasattr(event, "kwargs"): + kwargs.update(event.kwargs) + + room = None + passed_room = None + if "room" in kwargs: + passed_room = kwargs["room"] + if "channel" in kwargs: + passed_room = kwargs["channel"] + + if passed_room: + if isinstance(passed_room, str): + # User passed in a room string + room = self.get_room_from_name_or_id(passed_room) + else: + # User found the internal HipChatRoom object and passed it. + room = passed_room + else: + # Default to the room we heard this message in. + room = self.get_room_from_message(event) + + room_id = None + if room and hasattr(room, "id"): + room_id = room.id + else: + room_id = room + + if event.type in ["say", "reply"]: + event.content = re.sub(r'>\s+<', '><', event.content) + + if hasattr(event, "source_message") and event.source_message and not room: + send_source = event.source_message + + if hasattr(event.source_message, "data"): + send_source = event.source_message.data + + if send_source.is_private_chat: + # Private, 1-1 chats. + self.send_direct_message(send_source.sender.id, event.content, **kwargs) + return + + # Otherwise trust room. + self.send_room_message( + room_id, + event.content, + **kwargs + ) + + elif event.type in ["topic_change", ]: + if room_id: + self.set_room_topic(room_id, event.content) + else: + if hasattr(event, "source_message") and event.source_message: + send_source = event.source_message + + if hasattr(event.source_message, "data"): + send_source = event.source_message.data + self.send_direct_message(send_source.sender.id, "I can't set the topic of a one-to-one chat. Let's just talk.", **kwargs) + + elif ( + event.type == "message.no_response" and + event.data.is_direct and + event.data.will_said_it is False + ): + if event.data.original_incoming_event.type == "groupchat": + self.send_room_message( + event.data.channel.id, + random.choice(UNSURE_REPLIES), + **kwargs + ) + else: + self.send_direct_message( + event.data.sender.id, + random.choice(UNSURE_REPLIES), + **kwargs + ) + + def __handle_bridge_queue(self): + while True: + try: + try: + input_event = self.xmpp_bridge_queue.get(timeout=settings.EVENT_LOOP_INTERVAL) + if input_event: + self.handle_incoming_event(input_event) + + except Empty: + pass + + except (KeyboardInterrupt, SystemExit): + pass + self.sleep_for_event_loop() + + def bootstrap(self): + # Bootstrap must provide a way to to have: + # a) self.normalize_incoming_event fired, or incoming events put into self.incoming_queue + # b) any necessary threads running for a) + # c) self.me (Person) defined, with Will's info + # d) self.people (dict of People) defined, with everyone in an organization/backend + # e) self.channels (dict of Channels) defined, with all available channels/rooms. + # Note that Channel asks for members, a list of People. + # f) A way for self.handle, self.me, self.people, and self.channels to be kept accurate, + # with a maximum lag of 60 seconds. + self.client = HipChatXMPPClient("%s/bot" % settings.HIPCHAT_USERNAME, settings.HIPCHAT_PASSWORD) + self.xmpp_bridge_queue = Queue() + self.client.start_xmpp_client( + xmpp_bridge_queue=self.xmpp_bridge_queue, + backend_name=self.internal_name, + ) + self.client.connect() + # Even though these are properties, they do some gets and self-fillings. + self.people + self.channels + + self.bridge_thread = Process(target=self.__handle_bridge_queue) + self.bridge_thread.start() + self.xmpp_thread = Process(target=self.client.process, kwargs={"block": True}) + self.xmpp_thread.start() + + def terminate(self): + if hasattr(self, "xmpp_thread"): + self.xmpp_thread.terminate() + if hasattr(self, "bridge_thread"): + self.bridge_thread.terminate() + + while ( + (hasattr(self, "xmpp_thread") and self.xmpp_thread.is_alive()) or + (hasattr(self, "bridge_thread") and self.bridge_thread.is_alive()) + ): + time.sleep(0.2) diff --git a/will/backends/io_adapters/rocketchat.py b/will/backends/io_adapters/rocketchat.py new file mode 100644 index 00000000..69ad703f --- /dev/null +++ b/will/backends/io_adapters/rocketchat.py @@ -0,0 +1,499 @@ +import ctypes +import json +import html2text +import logging +import pprint +import random +import requests +import sys +import time +import traceback + +from DDPClient import DDPClient +from multiprocessing import Manager +from multiprocessing.dummy import Process +from six.moves.urllib import parse + +from will import settings +from will.abstractions import Event, Message, Person, Channel +from will.mixins import SleepMixin, StorageMixin +from will.utils import Bunch, UNSURE_REPLIES, clean_for_pickling +from .base import IOBackend + + +class RocketChatBackend(IOBackend, StorageMixin): + friendly_name = "RocketChat" + internal_name = "will.backends.io_adapters.rocketchat" + required_settings = [ + { + "name": "ROCKETCHAT_USERNAME", + "obtain_at": """1. Go to your rocket.chat instance (i.e. your-name.rocket.chat) +2. Create a new normal account for Will. +3. Set this value to the username, just like you'd use to log in with it.""", + }, + { + "name": "ROCKETCHAT_PASSWORD", + "obtain_at": """1. Go to your rocket.chat instance (i.e. your-name.rocket.chat) +2. Create a new normal account for Will, and note the password you use. +3. Set this value to that password, just like you'd use to log in with it.""", + }, + { + "name": "ROCKETCHAT_URL", + "obtain_at": ( + "This is your rocket.chat url - typically either your-name.rocket.chat for " + "Rocket.Chat cloud, or something like http://localhost:3000 for local installations." + ), + }, + ] + + pp = pprint.PrettyPrinter(indent=4) + + def normalize_incoming_event(self, event): + logging.info('Normalizing incoming Rocket.Chat event') + logging.debug('event: {}'.format(self.pp.pformat(event))) + if event["type"] == "message": + + # Were we mentioned? + will_is_mentioned = False + for mention in event['mentions']: + if mention['username'] == self.me.handle: + will_is_mentioned = True + break + + # Handle direct messages, which in Rocket.Chat are a rid + # made up of both users' _ids. + is_private_chat = False + if self.me.id in event["rid"]: + is_private_chat = True + + # Create a "Channel" to align with Rocket.Chat DM + # paradigm. There might well be a better way of doing + # this. See TODO in _rest_channels_list. + sender_id = event['u']['_id'] + ids = [sender_id, self.me.id] + ids.sort() + channel_id = '{}{}'.format(*ids) + sender = self.people[sender_id] + channel_members = {} + channel_members[sender_id] = sender + channel_members[self.me.id] = self.me + channel = Channel( + id=channel_id, + name=channel_id, + source=clean_for_pickling(channel_id), + members=channel_members + ) + else: + if "rid" in event and event["rid"] in self.channels: + channel = clean_for_pickling(self.channels[event["rid"]]) + else: + # Private channel, unknown members. Just do our best and try to route it. + if "rid" in event: + channel = Channel( + id=event["rid"], + name=event["rid"], + source=clean_for_pickling(event["rid"]), + members={} + ) + logging.debug('channel: {}'.format(channel)) + + # Set various variables depending on whether @handle was + # part of the message. + interpolated_handle = "@%s " % self.handle + logging.debug('interpolated_handle: {}' + .format(interpolated_handle)) + is_direct = False + if is_private_chat or event['msg'].startswith(interpolated_handle): + is_direct = True + + # Strip my handle from the start. NB Won't strip it from + # elsewhere in the text, and won't strip other mentions. + # This will stop regexes from working, not sure if it's a + # feature or a bug. + if event['msg'].startswith(interpolated_handle): + event['msg'] = event['msg'][len(interpolated_handle):].strip() + + if interpolated_handle in event['msg']: + will_is_mentioned = True + + # Determine if Will said it. + logging.debug('self.people: {}'.format(self.pp.pformat(self.people))) + sender = self.people[event['u']['_id']] + logging.debug('sender: {}'.format(sender)) + if sender['handle'] == self.me.handle: + logging.debug('Will said it') + will_said_it = True + else: + logging.debug('Will didnt say it') + will_said_it = False + + m = Message( + content=event['msg'], + type=event.type, + is_direct=is_direct, + is_private_chat=is_private_chat, + is_group_chat=not is_private_chat, + backend=self.internal_name, + sender=sender, + channel=channel, + will_is_mentioned=will_is_mentioned, + will_said_it=will_said_it, + backend_supports_acl=True, + original_incoming_event=clean_for_pickling(event) + ) + return m + else: + logging.debug('Passing, I dont know how to normalize this event of type ', event["type"]) + pass + + def handle_outgoing_event(self, event): + # Print any replies. + logging.info('Handling outgoing Rocket.Chat event') + logging.debug('event: {}'.format(self.pp.pformat(event))) + + if event.type in ["say", "reply"]: + + if "kwargs" in event and "html" in event.kwargs and event.kwargs["html"]: + event.content = html2text.html2text(event.content) + + self.send_message(event) + if hasattr(event, "source_message") and event.source_message: + pass + else: + # Backend needs to provide ways to handle and properly route: + # 1. 1-1 messages + # 2. Group (channel) messages + # 3. Ad-hoc group messages (if they exist) + # 4. Messages that have a channel/room explicitly specified that's different than + # where they came from. + # 5. Messages without a channel (Fallback to ROCKETCHAT_DEFAULT_CHANNEL) (messages that don't have a room ) + kwargs = {} + if "kwargs" in event: + kwargs.update(**event.kwargs) + + if event.type in ["topic_change", ]: + self.set_topic(event.content) + elif ( + event.type == "message.no_response" and + event.data.is_direct and + event.data.will_said_it is False + ): + event.content = random.choice(UNSURE_REPLIES) + self.send_message(event) + + def set_topic(self, event): + logging.warn("Rocket.Chat doesn't support topics yet: https://github.com/RocketChat/Rocket.Chat/issues/328") + event.content("Hm. Looks like Rocket.Chat doesn't support topics yet: https://github.com/RocketChat/Rocket.Chat/issues/328") + self.send_message(event) + + def send_message(self, event): + logging.info('Sending message to Rocket.Chat') + logging.debug('event: {}'.format(self.pp.pformat(event))) + data = {} + if hasattr(event, "kwargs"): + logging.debug('event.kwargs: {}'.format(event.kwargs)) + data.update(event.kwargs) + + # TODO: Go through the possible attachment parameters at + # https://rocket.chat/docs/developer-guides/rest-api/chat/postmessage + # - this is a bare minimum inspired by slack.py + if 'color' in event.kwargs: + data.update({ + "attachments": [ + { + 'color': event.kwargs["color"], + 'text': event.content, + } + ], + }) + else: + data.update({ + 'text': event.content, + }) + else: + # I haven't seen this yet, not sure when it's relevant. + # 'text' was wrongly set to 'msg' and nothing blew up. ;) + logging.debug("event doesn't have kwargs") + data.update({ + 'text': event.content, + }) + + if "source_message" in event: + if hasattr(event.source_message, "data"): + data['roomId'] = event.source_message.data.channel.id + else: + data['roomId'] = event.source_message.channel.id + else: + data['roomId'] = event.data['source'].data.channel.id + + self._rest_post_message(data) + + def _get_rest_metadata(self): + self._rest_users_list() + self._rest_channels_list() + + def _get_realtime_metadata(self): + self._realtime_get_rooms() + + # REST API functions, documented at + # https://rocket.chat/docs/developer-guides/rest-api/ + + def _rest_login(self): + params = {'username': settings.ROCKETCHAT_USERNAME, + 'password': settings.ROCKETCHAT_PASSWORD} + r = requests.post('{}login'.format(self.rocketchat_api_url), + data=params) + resp_json = r.json() + self._token = resp_json['data']['authToken'] + self.save("WILL_ROCKETCHAT_TOKEN", self._token) + self._userid = resp_json['data']['userId'] + self.save("WILL_ROCKETCHAT_USERID", self._userid) + + def _rest_users_list(self): + logging.debug('Getting users list from Rocket.Chat') + + # Remember to paginate. ;) + count = 50 + passes = 0 + headers = {'X-Auth-Token': self.token, + 'X-User-Id': self.userid} + fetched = 0 + total = 0 + + self.handle = settings.ROCKETCHAT_USERNAME + self.mention_handle = "@%s" % settings.ROCKETCHAT_USERNAME + + people = {} + + while fetched <= total: + params = {'count': count, + 'offset': fetched} + r = requests.get('{}users.list'.format(self.rocketchat_api_url), + headers=headers, + params=params) + resp_json = r.json() + if resp_json['success'] is False: + logging.exception('resp_json: {}'.format(resp_json)) + total = resp_json['total'] + + for user in resp_json['users']: + # TODO: Unlike slack.py, no timezone support at present. + # RC returns utcOffset, but this isn't enough to + # determine timezone. + # TODO: Pickle error if timezone set to UTC, and I didn't + # have a chance to report it. Using GMT as a poor substitute. + person = Person( + id=user['_id'], + handle=user['username'], + mention_handle="@%s" % user["username"], + source=clean_for_pickling(user)['username'], + name=user['name'], + timezone='GMT' + ) + + people[user['_id']] = person + if user['username'] == self.handle: + self.me = person + + passes += 1 + fetched = count * passes + + self.people = people + + def _get_userid_from_username(self, username): + if username is None: + raise TypeError("No username given") + + for id, data in self.people.items(): + if data['handle'] == username: + return id + + def _rest_channels_list(self): + logging.debug('Getting channel list from Rocket.Chat') + + # Remember to paginate. ;) + count = 50 + passes = 0 + headers = {'X-Auth-Token': self.token, + 'X-User-Id': self.userid} + fetched = 0 + total = 0 + + channels = {} + + while fetched <= total: + r = requests.get('{}channels.list'.format(self.rocketchat_api_url), + headers=headers) + resp_json = r.json() + + total = resp_json['total'] + + for channel in resp_json['channels']: + members = {} + for username in channel['usernames']: + userid = self._get_userid_from_username(username) + members[userid] = self.people[userid] + + channels[channel['_id']] = Channel( + id=channel['_id'], + name=channel['name'], + source=clean_for_pickling(channel), + members=members + ) + + passes += 1 + fetched = count * passes + + self.channels = channels + + def _rest_post_message(self, data): + logging.info('Posting message to Rocket.Chat REST API') + logging.debug('data: {}'.format(data)) + headers = { + 'X-Auth-Token': self.token, + 'X-User-Id': self.userid + } + logging.debug('headers: {}'.format(headers)) + r = requests.post( + '{}chat.postMessage'.format(self.rocketchat_api_url), + headers=headers, + data=data, + ) + resp_json = r.json() + + # TODO: Necessary / useful to check return codes? + if not 'success' in resp_json: + logging.debug('resp_json: {}'.format(resp_json)) + assert resp_json['success'] + + # Realtime API functions, documented at + # https://rocket.chat/docs/developer-guides/realtime-api/ + + def _start_connect(self): + up = parse.urlparse(settings.ROCKETCHAT_URL) + if up.scheme == 'http': + ws_proto = 'ws' + else: + ws_proto = 'wss' + self.rc = DDPClient('{}://{}/websocket'.format(ws_proto, up.netloc), auto_reconnect=True, auto_reconnect_timeout=1) + self.rc.on('connected', self._realtime_login) + self.rc.on('changed', self._changed_callback) + self.rc.connect() + + def _realtime_login(self): + params = [{'user': {'username': settings.ROCKETCHAT_USERNAME}, 'password': settings.ROCKETCHAT_PASSWORD}] + self.rc.call('login', params, self._login_callback) + + def _login_callback(self, error, result): + logging.debug('_login_callback') + if error: + logging.exception('error: {}'.format(error)) + return + + logging.debug('result: {}'.format(result)) + logging.debug('self.token: {}'.format(self.token)) + logging.debug('self.userid: {}'.format(self.userid)) + + # Use dummy to make it a Thread, otherwise DDP events don't + # get back to the right place. If there is a real need to make + # it a real Process, it is probably just a matter of using + # multiprocessing.Value(s) in the right place(s). + # TODO: Could this be the reason for the 100% CPU usage? + # Have asked in #development. + self.update_thread = Process(target=self._get_updates) + self.update_thread.start() + + def _changed_callback(self, collection, _id, fields, cleared): + logging.debug('_changed_callback') + logging.debug('collection: {}'.format(collection)) + logging.debug('id: {}'.format(_id)) + logging.debug('fields: {}'.format(self.pp.pformat(fields))) + logging.debug('cleared: {}'.format(cleared)) + event = Event(type='message', version=1, **fields['args'][0]) + self.handle_incoming_event(event) + + def _stream_room_message_callback(self, error, event): + logging.debug('_stream_room_message_callback') + if error: + logging.exception('error: {}'.format(error)) + return + + @property + def token(self): + if not hasattr(self, "_token") or not self._token: + self._token = self.load("WILL_ROCKETCHAT_TOKEN", None) + if not self._token: + self._rest_login() + return self._token + + @property + def userid(self): + if not hasattr(self, "_userid") or not self._userid: + self._userid = self.load("WILL_ROCKETCHAT_USERID", None) + if not self._userid: + self._rest_login() + return self._userid + + @property + def rocketchat_api_url(self): + if settings.ROCKETCHAT_URL.endswith("/"): + return settings.ROCKETCHAT_URL + 'api/v1/' + else: + return settings.ROCKETCHAT_URL + '/api/v1/' + + # Gets updates from REST and Realtime APIs. + def _get_updates(self): + try: + polling_interval_seconds = 5 + self._get_rest_metadata() + + while True: + # Update channels/people/me/etc. + self._get_rest_metadata() + self._get_realtime_metadata() + + time.sleep(polling_interval_seconds) + except (KeyboardInterrupt, SystemExit): + pass + except: + logging.critical("Error in watching RocketChat API: \n%s" % traceback.format_exc()) + + # Use this to get a list of all rooms that we are in. + # https://rocket.chat/docs/developer-guides/realtime-api/the-room-object + def _realtime_get_rooms(self): + params = [{'$date': 0}] + self.rc.call('rooms/get', params, self._get_rooms_callback) + + def _get_rooms_callback(self, error, result): + logging.debug('_get_rooms_callback') + if error: + logging.exception('_get_rooms_callback error: {}'.format(error)) + return + + # TODO: When we leave a room, we don't delete it from + # self.subscribed_rooms. Not a problem in practice - + # subscriptions to the room won't fire, but messy. + for room in result: + logging.debug('room: {}'.format(room)) + if room['_id'] not in self.subscribed_rooms: + self.rc.subscribe('stream-room-messages', [room['_id']], + self._stream_room_message_callback) + self.subscribed_rooms[room['_id']] = True + + def bootstrap(self): + # Bootstrap must provide a way to to have: + # a) self.normalize_incoming_event fired, or incoming events put into self.incoming_queue + # b) any necessary threads running for a) + # c) self.me (Person) defined, with Will's info + # d) self.people (dict of People) defined, with everyone in an organization/backend + # e) self.channels (dict of Channels) defined, with all available channels/rooms. + # Note that Channel asks for users, a list of People. + # f) A way for self.handle, self.me, self.people, and self.channels to be kept accurate, + # with a maximum lag of 60 seconds. + + self.subscribed_rooms = {} + + # Gets and stores token and ID. + self._rest_login() + # Kicks off listeners and REST room polling. + self._start_connect() diff --git a/will/backends/io_adapters/shell.py b/will/backends/io_adapters/shell.py new file mode 100644 index 00000000..16b57c61 --- /dev/null +++ b/will/backends/io_adapters/shell.py @@ -0,0 +1,110 @@ +import cmd +import random +import sys +import time +import logging +import requests +import threading +import readline +import traceback +import warnings + +from will import settings +from will.utils import Bunch, UNSURE_REPLIES, html_to_text +from will.abstractions import Message, Person, Channel +from .base import StdInOutIOBackend + +warnings.filterwarnings("ignore", category=UserWarning, module='bs4') + + +class ShellBackend(StdInOutIOBackend): + friendly_name = "Interactive Shell" + internal_name = "will.backends.io_adapters.shell" + partner = Person( + id="you", + handle="shelluser", + mention_handle="@shelluser", + source=Bunch(), + name="Friend", + ) + + def send_direct_message(self, message_body, **kwargs): + print("Will: %s" % html_to_text(message_body)) + + def send_room_message(self, room_id, message_body, html=False, color="green", notify=False, **kwargs): + print("Will: %s" % html_to_text(message_body)) + + def set_room_topic(self, topic): + print("Will: Let's talk about %s" % (topic, )) + + def normalize_incoming_event(self, event): + if event["type"] == "message.incoming.stdin": + m = Message( + content=event.data.content.strip(), + type=event.type, + is_direct=True, + is_private_chat=True, + is_group_chat=False, + backend=self.internal_name, + sender=self.partner, + will_is_mentioned=False, + will_said_it=False, + backend_supports_acl=False, + original_incoming_event=event + ) + return m + else: + # An event type the shell has no idea how to handle. + return None + + def handle_outgoing_event(self, event): + # Print any replies. + if event.type in ["say", "reply"]: + self.send_direct_message(event.content) + if event.type in ["topic_change", ]: + self.set_room_topic(event.content) + + elif event.type == "message.no_response": + if event.data and hasattr(event.data, "original_incoming_event") and len(event.data.original_incoming_event.data.content) > 0: + self.send_direct_message(random.choice(UNSURE_REPLIES)) + + # Regardless of whether or not we had something to say, + # give the user a new prompt. + sys.stdout.write("You: ") + sys.stdout.flush() + + def bootstrap(self): + # Bootstrap must provide a way to to have: + # a) self.normalize_incoming_event fired, or incoming events put into self.incoming_queue + # b) any necessary threads running for a) + # c) self.me (Person) defined, with Will's info + # d) self.people (dict of People) defined, with everyone in an organization/backend + # e) self.channels (dict of Channels) defined, with all available channels/rooms. + # Note that Channel asks for members, a list of People. + # f) A way for self.handle, self.me, self.people, and self.channels to be kept accurate, + # with a maximum lag of 60 seconds. + self.people = {} + self.channels = {} + self.me = Person( + id="will", + handle="will", + mention_handle="@will", + source=Bunch(), + name="William T. Botterton", + ) + + # Do this to get the first "you" prompt. + self.pubsub.publish('message.incoming.stdin', (Message( + content="", + type="message.incoming", + is_direct=True, + is_private_chat=True, + is_group_chat=False, + backend=self.internal_name, + sender=self.partner, + will_is_mentioned=False, + will_said_it=False, + backend_supports_acl=False, + original_incoming_event={} + )) + ) diff --git a/will/backends/io_adapters/slack.py b/will/backends/io_adapters/slack.py new file mode 100644 index 00000000..3dc40060 --- /dev/null +++ b/will/backends/io_adapters/slack.py @@ -0,0 +1,547 @@ +import json +import logging +import random +import re +import requests +import sys +import time +import traceback +from websocket import WebSocketConnectionClosedException + +from markdownify import MarkdownConverter + +from will import settings +from .base import IOBackend +from will.utils import Bunch, UNSURE_REPLIES, clean_for_pickling +from will.mixins import SleepMixin, StorageMixin +from multiprocessing import Process +from will.abstractions import Event, Message, Person, Channel +from slackclient import SlackClient +from slackclient.server import SlackConnectionError + +SLACK_SEND_URL = "https://slack.com/api/chat.postMessage" +SLACK_SET_TOPIC_URL = "https://slack.com/api/channels.setTopic" +SLACK_PRIVATE_SET_TOPIC_URL = "https://slack.com/api/groups.setTopic" + + +class SlackMarkdownConverter(MarkdownConverter): + + def convert_strong(self, el, text): + return '*%s*' % text if text else '' + + +class SlackBackend(IOBackend, SleepMixin, StorageMixin): + friendly_name = "Slack" + internal_name = "will.backends.io_adapters.slack" + required_settings = [ + { + "name": "SLACK_API_TOKEN", + "obtain_at": """1. Go to https://api.slack.com/custom-integrations/legacy-tokens and sign in as yourself (or a user for Will). +2. Find the workspace you want to use, and click "Create token." +3. Set this token as SLACK_API_TOKEN.""" + } + ] + + def get_channel_from_name(self, name): + for k, c in self.channels.items(): + if c.name.lower() == name.lower() or c.id.lower() == name.lower(): + return c + + def normalize_incoming_event(self, event): + + if ( + "type" in event and + event["type"] == "message" and + ("subtype" not in event or event["subtype"] != "message_changed") and + # Ignore thread summary events (for now.) + # TODO: We should stack these into the history. + ("subtype" not in event or ("message" in event and "thread_ts" not in event["message"])) + ): + # print("slack: normalize_incoming_event - %s" % event) + # Sample of group message + # {u'source_team': u'T5ACF70KV', u'text': u'test', + # u'ts': u'1495661121.838366', u'user': u'U5ACF70RH', + # u'team': u'T5ACF70KV', u'type': u'message', u'channel': u'C5JDAR2S3'} + + # Sample of 1-1 message + # {u'source_team': u'T5ACF70KV', u'text': u'test', + # u'ts': u'1495662397.335424', u'user': u'U5ACF70RH', + # u'team': u'T5ACF70KV', u'type': u'message', u'channel': u'D5HGP0YE7'} + + # Threaded message + # {u'event_ts': u'1507601477.000073', u'ts': u'1507601477.000073', + # u'subtype': u'message_replied', u'message': + # {u'thread_ts': u'1507414046.000010', u'text': u'hello!', + # u'ts': u'1507414046.000010', u'unread_count': 2, + # u'reply_count': 2, u'user': u'U5GUL9D9N', u'replies': + # [{u'user': u'U5ACF70RH', u'ts': u'1507601449.000007'}, { + # u'user': u'U5ACF70RH', u'ts': u'1507601477.000063'}], + # u'type': u'message', u'bot_id': u'B5HL9ABFE'}, + # u'type': u'message', u'hidden': True, u'channel': u'D5HGP0YE7'} + + sender = self.people[event["user"]] + channel = clean_for_pickling(self.channels[event["channel"]]) + # print "channel: %s" % channel + interpolated_handle = "<@%s>" % self.me.id + real_handle = "@%s" % self.me.handle + will_is_mentioned = False + will_said_it = False + + is_private_chat = False + + thread = None + if "thread_ts" in event: + thread = event["thread_ts"] + + # If the parent thread is a 1-1 between Will and I, also treat that as direct. + # Since members[] still comes in on the thread event, we can trust this, even if we're + # in a thread. + if channel.id == channel.name: + is_private_chat = True + + # <@U5GUL9D9N> hi + # TODO: if there's a thread with just will and I on it, treat that as direct. + is_direct = False + if is_private_chat or event["text"].startswith(interpolated_handle) or event["text"].startswith(real_handle): + is_direct = True + + if event["text"].startswith(interpolated_handle): + event["text"] = event["text"][len(interpolated_handle):].strip() + + if event["text"].startswith(real_handle): + event["text"] = event["text"][len(real_handle):].strip() + + if interpolated_handle in event["text"] or real_handle in event["text"]: + will_is_mentioned = True + + if event["user"] == self.me.id: + will_said_it = True + + m = Message( + content=event["text"], + type=event["type"], + is_direct=is_direct, + is_private_chat=is_private_chat, + is_group_chat=not is_private_chat, + backend=self.internal_name, + sender=sender, + channel=channel, + thread=thread, + will_is_mentioned=will_is_mentioned, + will_said_it=will_said_it, + backend_supports_acl=True, + original_incoming_event=clean_for_pickling(event), + ) + return m + else: + # An event type the slack ba has no idea how to handle. + pass + + def set_topic(self, event): + headers = {'Accept': 'text/plain'} + data = self.set_data_channel_and_thread(event) + data.update({ + "token": settings.SLACK_API_TOKEN, + "as_user": True, + "topic": event.content, + }) + if data["channel"].startswith("G"): + url = SLACK_PRIVATE_SET_TOPIC_URL + else: + url = SLACK_SET_TOPIC_URL + r = requests.post( + url, + headers=headers, + data=data, + **settings.REQUESTS_OPTIONS + ) + self.handle_request(r, data) + + def handle_outgoing_event(self, event): + if event.type in ["say", "reply"]: + if "kwargs" in event and "html" in event.kwargs and event.kwargs["html"]: + event.content = SlackMarkdownConverter().convert(event.content) + + event.content = event.content.replace("&", "&") + event.content = event.content.replace("\_", "_") + + kwargs = {} + if "kwargs" in event: + kwargs.update(**event.kwargs) + + if hasattr(event, "source_message") and event.source_message and "channel" not in kwargs: + self.send_message(event) + else: + # Came from webhook/etc + # TODO: finish this. + target_channel = kwargs.get("room", kwargs.get("channel", None)) + if target_channel: + event.channel = self.get_channel_from_name(target_channel) + if event.channel: + self.send_message(event) + else: + logging.error( + "I was asked to post to the slack %s channel, but it doesn't exist.", + target_channel + ) + if self.default_channel: + event.channel = self.get_channel_from_name(self.default_channel) + event.content = event.content + " (for #%s)" % target_channel + self.send_message(event) + + elif self.default_channel: + event.channel = self.get_channel_from_name(self.default_channel) + self.send_message(event) + else: + logging.critical( + "I was asked to post to a slack default channel, but I'm nowhere." + "Please invite me somewhere with '/invite @%s'", self.me.handle + ) + + if event.type in ["topic_change", ]: + self.set_topic(event) + elif ( + event.type == "message.no_response" and + event.data.is_direct and + event.data.will_said_it is False + ): + event.content = random.choice(UNSURE_REPLIES) + self.send_message(event) + + def handle_request(self, r, data): + resp_json = r.json() + if not resp_json["ok"]: + if resp_json["error"] == "not_in_channel": + channel = self.get_channel_from_name(data["channel"]) + if not hasattr(self, "me") or not hasattr(self.me, "handle"): + self.people + + logging.critical( + "I was asked to post to the slack %s channel, but I haven't been invited. " + "Please invite me with '/invite @%s'" % (channel.name, self.me.handle) + ) + else: + logging.error("Error sending to slack: %s" % resp_json["error"]) + logging.error(resp_json) + assert resp_json["ok"] + + def set_data_channel_and_thread(self, event, data={}): + if "channel" in event: + # We're coming off an explicit set. + channel_id = event.channel.id + else: + if "source_message" in event: + # Mentions that come back via self.say() + if hasattr(event.source_message, "data"): + channel_id = event.source_message.data.channel.id + if hasattr(event.source_message.data, "thread"): + data.update({ + "thread_ts": event.source_message.data.thread + }) + else: + # Mentions that come back via self.say() with a specific room (I think) + channel_id = event.source_message.channel.id + if hasattr(event.source_message, "thread"): + data.update({ + "thread_ts": event.source_message.thread + }) + else: + # Mentions that come back via self.reply() + if hasattr(event.data, "original_incoming_event"): + if hasattr(event.data.original_incoming_event.channel, "id"): + channel_id = event.data.original_incoming_event.channel.id + else: + channel_id = event.data.original_incoming_event.channel + else: + if hasattr(event.data["original_incoming_event"].data.channel, "id"): + channel_id = event.data["original_incoming_event"].data.channel.id + else: + channel_id = event.data["original_incoming_event"].data.channel + + try: + # If we're starting a thread + if "kwargs" in event and "start_thread" in event.kwargs and event.kwargs["start_thread"] and ("thread_ts" not in data or not data["thread_ts"]): + if hasattr(event.source_message, "original_incoming_event"): + data.update({ + "thread_ts": event.source_message.original_incoming_event["ts"] + }) + elif ( + hasattr(event.source_message, "data") and + hasattr(event.source_message.data, "original_incoming_event") and + "ts" in event.source_message.data.original_incoming_event + ): + logging.error( + "Hm. I was told to start a new thread, but while using .say(), instead of .reply().\n" + "This doesn't really make sense, but I'm going to make the best of it by pretending you " + "used .say() and threading off of your message.\n" + "Please update your plugin to use .reply() when you have a second!" + ) + data.update({ + "thread_ts": event.source_message.data.original_incoming_event["ts"] + }) + else: + if hasattr(event.data.original_incoming_event, "thread_ts"): + data.update({ + "thread_ts": event.data.original_incoming_event.thread_ts + }) + elif "thread" in event.data.original_incoming_event.data: + data.update({ + "thread_ts": event.data.original_incoming_event.data.thread + }) + except: + logging.info(traceback.format_exc().split(" ")[-1]) + pass + data.update({ + "channel": channel_id, + }) + return data + + def send_message(self, event): + data = {} + if hasattr(event, "kwargs"): + data.update(event.kwargs) + + # Add slack-specific functionality + if "color" in event.kwargs: + data.update({ + "attachments": json.dumps([ + { + "fallback": event.content, + "color": self._map_color(event.kwargs["color"]), + "text": event.content, + } + ]), + }) + elif "attachments" in event.kwargs: + data.update({ + "text": event.content, + "attachments": json.dumps(event.kwargs["attachments"]) + }) + else: + data.update({ + "text": event.content, + }) + else: + data.update({ + "text": event.content, + }) + + data = self.set_data_channel_and_thread(event, data=data) + + # Auto-link mention names + if "text" in data: + if data["text"].find("<@") != -1: + data["text"] = data["text"].replace("<@", "<@") + data["text"] = data["text"].replace(">", ">") + elif "attachments" in data and "text" in data["attachments"][0]: + if data["attachments"][0]["text"].find("<@") != -1: + data["attachments"][0]["text"] = data["attachments"][0]["text"].replace("<@", "<@") + data["attachments"][0]["text"] = data["attachments"][0]["text"].replace(">", ">") + + data.update({ + "token": settings.SLACK_API_TOKEN, + "as_user": True, + }) + if hasattr(event, "kwargs") and "html" in event.kwargs and event.kwargs["html"]: + data.update({ + "parse": "full", + }) + + headers = {'Accept': 'text/plain'} + r = requests.post( + SLACK_SEND_URL, + headers=headers, + data=data, + **settings.REQUESTS_OPTIONS + ) + self.handle_request(r, data) + + def _map_color(self, color): + # Turn colors into hex values, handling old slack colors, etc + if color == "red": + return "danger" + elif color == "yellow": + return "warning" + elif color == "green": + return "good" + + return color + + def join_channel(self, channel_id): + return self.client.api_call( + "channels.join", + channel=channel_id, + ) + + @property + def people(self): + if not hasattr(self, "_people") or self._people is {}: + self._update_people() + return self._people + + @property + def default_channel(self): + if not hasattr(self, "_default_channel") or not self._default_channel: + self._decide_default_channel() + return self._default_channel + + @property + def channels(self): + if not hasattr(self, "_channels") or self._channels is {}: + self._update_channels() + return self._channels + + @property + def client(self): + if not hasattr(self, "_client"): + self._client = SlackClient(settings.SLACK_API_TOKEN) + return self._client + + def _decide_default_channel(self): + self._default_channel = None + if not hasattr(self, "complained_about_default"): + self.complained_about_default = False + self.complained_uninvited = False + + # Set self.me + self.people + + if hasattr(settings, "SLACK_DEFAULT_CHANNEL"): + channel = self.get_channel_from_name(settings.SLACK_DEFAULT_CHANNEL) + if channel: + if self.me.id in channel.members: + self._default_channel = channel.id + return + elif not self.complained_about_default: + self.complained_about_default = True + logging.error("The defined default channel(%s) does not exist!", + settings.SLACK_DEFAULT_CHANNEL) + + for c in self.channels.values(): + if c.name != c.id and self.me.id in c.members: + self._default_channel = c.id + if not self._default_channel and not self.complained_uninvited: + self.complained_uninvited = True + logging.critical("No channels with me invited! No messages will be sent!") + + def _update_channels(self): + channels = {} + for c in self.client.server.channels: + members = {} + for m in c.members: + members[m] = self.people[m] + + channels[c.id] = Channel( + id=c.id, + name=c.name, + source=clean_for_pickling(c), + members=members + ) + if len(channels.keys()) == 0: + # Server isn't set up yet, and we're likely in a processing thread, + if self.load("slack_channel_cache", None): + self._channels = self.load("slack_channel_cache", None) + else: + self._channels = channels + self.save("slack_channel_cache", channels) + + def _update_people(self): + people = {} + + self.handle = self.client.server.username + + for k, v in self.client.server.users.items(): + user_timezone = None + if v.tz: + user_timezone = v.tz + people[k] = Person( + id=v.id, + mention_handle="<@%s>" % v.id, + handle=v.name, + source=clean_for_pickling(v), + name=v.real_name, + ) + if v.name == self.handle: + self.me = Person( + id=v.id, + mention_handle="<@%s>" % v.id, + handle=v.name, + source=clean_for_pickling(v), + name=v.real_name, + ) + if user_timezone and user_timezone != 'unknown': + people[k].timezone = user_timezone + if v.name == self.handle: + self.me.timezone = user_timezone + if len(people.keys()) == 0: + # Server isn't set up yet, and we're likely in a processing thread, + if self.load("slack_people_cache", None): + self._people = self.load("slack_people_cache", None) + if not hasattr(self, "me") or not self.me: + self.me = self.load("slack_me_cache", None) + if not hasattr(self, "handle") or not self.handle: + self.handle = self.load("slack_handle_cache", None) + else: + self._people = people + self.save("slack_people_cache", people) + self.save("slack_me_cache", self.me) + self.save("slack_handle_cache", self.handle) + + def _update_backend_metadata(self): + self._update_people() + self._update_channels() + + def _watch_slack_rtm(self): + while True: + try: + if self.client.rtm_connect(auto_reconnect=True): + self._update_backend_metadata() + + num_polls_between_updates = 30 / settings.EVENT_LOOP_INTERVAL # Every 30 seconds + current_poll_count = 0 + while True: + events = self.client.rtm_read() + if len(events) > 0: + # TODO: only handle events that are new. + # print(len(events)) + for e in events: + self.handle_incoming_event(e) + + # Update channels/people/me/etc every 10s or so. + current_poll_count += 1 + if current_poll_count > num_polls_between_updates: + self._update_backend_metadata() + current_poll_count = 0 + + self.sleep_for_event_loop() + except (WebSocketConnectionClosedException, SlackConnectionError): + logging.error('Encountered connection error attempting reconnect in 2 seconds') + time.sleep(2) + except (KeyboardInterrupt, SystemExit): + break + except: + logging.critical("Error in watching slack RTM: \n%s" % traceback.format_exc()) + break + + def bootstrap(self): + # Bootstrap must provide a way to to have: + # a) self.normalize_incoming_event fired, or incoming events put into self.incoming_queue + # b) any necessary threads running for a) + # c) self.me (Person) defined, with Will's info + # d) self.people (dict of People) defined, with everyone in an organization/backend + # e) self.channels (dict of Channels) defined, with all available channels/rooms. + # Note that Channel asks for members, a list of People. + # f) A way for self.handle, self.me, self.people, and self.channels to be kept accurate, + # with a maximum lag of 60 seconds. + + # Property, auto-inits. + self.client + + self.rtm_thread = Process(target=self._watch_slack_rtm) + self.rtm_thread.start() + + def terminate(self): + if hasattr(self, "rtm_thread"): + self.rtm_thread.terminate() + while self.rtm_thread.is_alive(): + time.sleep(0.2) diff --git a/will/backends/pubsub/__init__.py b/will/backends/pubsub/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/will/backends/pubsub/base.py b/will/backends/pubsub/base.py new file mode 100644 index 00000000..12a9d109 --- /dev/null +++ b/will/backends/pubsub/base.py @@ -0,0 +1,125 @@ +import logging +import os +import redis +import traceback +from six.moves.urllib.parse import urlparse + +from will.abstractions import Event +from will import settings +from will.mixins import SettingsMixin, EncryptionMixin + +SKIP_TYPES = ["psubscribe", "punsubscribe", ] + + +class PubSubPrivateBase(SettingsMixin, EncryptionMixin): + """ + The private bits of the base pubsub backend. + """ + def __init__(self, *args, **kwargs): + self.recent_hashes = [] + + def publish(self, topic, obj, reference_message=None): + """ + Sends an object out over the pubsub connection, properly formatted, + and conforming to the protocol. Handles pickling for the wire, etc. + This method should *not* be subclassed. + """ + logging.debug("Publishing topic (%s): \n%s" % (topic, obj)) + e = Event( + data=obj, + type=topic, + ) + if hasattr(obj, "sender"): + e.sender = obj.sender + + if reference_message: + original_incoming_event_hash = None + if hasattr(reference_message, "original_incoming_event_hash"): + original_incoming_event_hash = reference_message.original_incoming_event_hash + elif hasattr(reference_message, "source") and hasattr(reference_message.source, "hash"): + original_incoming_event_hash = reference_message.source.hash + elif hasattr(reference_message, "source") and hasattr(reference_message.source, "original_incoming_event_hash"): + original_incoming_event_hash = reference_message.source.original_incoming_event_hash + elif hasattr(reference_message, "hash"): + original_incoming_event_hash = reference_message.hash + if original_incoming_event_hash: + e.original_incoming_event_hash = original_incoming_event_hash + + return self.publish_to_backend( + self._localize_topic(topic), + self.encrypt(e) + ) + + def unsubscribe(self, topic): + # This is mostly here for semantic consistency. + self.do_unsubscribe(topic) + + def _localize_topic(self, topic): + cleaned_topic = topic + if type(topic) == type([]): + cleaned_topic = [] + for t in topic: + if not t.startswith(settings.SECRET_KEY): + cleaned_topic.append("%s.%s" % (settings.SECRET_KEY, t)) + + elif not topic.startswith(settings.SECRET_KEY): + cleaned_topic = "%s.%s" % (settings.SECRET_KEY, topic) + return cleaned_topic + + def subscribe(self, topic): + return self.do_subscribe(self._localize_topic(topic)) + + def get_message(self): + """ + Gets the latest object from the backend, and handles unpickling + and validation. + """ + try: + m = self.get_from_backend() + if m and m["type"] not in SKIP_TYPES: + return self.decrypt(m["data"]) + + except AttributeError: + raise Exception("Tried to call get message without having subscribed first!") + except (KeyboardInterrupt, SystemExit): + pass + except: + logging.critical("Error in watching pubsub get message: \n%s" % traceback.format_exc()) + return None + + +class BasePubSub(PubSubPrivateBase): + """ + The base pubsub backend. + Subclassing methods must implement: + - do_subscribe() + - unsubscribe() + - publish_to_backend() + - get_from_backend() + """ + + def do_subscribe(self, topic): + """ + Registers with the backend to only get messages matching a specific topic. + Where possible, wildcards are allowed + """ + raise NotImplementedError + + def do_unsubscribe(self, topic): + """Unregisters with the backend for a given topic.""" + raise NotImplementedError + + def publish_to_backend(self, topic, str): + """Publishes a string to the backend with a given topic.""" + raise NotImplementedError + + def get_from_backend(self): + """ + Gets the latest pending message from the backend (FIFO). + Returns None if no messages are pending, and is expected *not* to be blocking. + """ + raise NotImplementedError + + +def bootstrap(settings): + return BasePubSub(settings) diff --git a/will/backends/pubsub/redis_pubsub.py b/will/backends/pubsub/redis_pubsub.py new file mode 100644 index 00000000..92ab4e55 --- /dev/null +++ b/will/backends/pubsub/redis_pubsub.py @@ -0,0 +1,69 @@ +import logging +import redis +from six.moves.urllib import parse +from .base import BasePubSub + +SKIP_TYPES = ["psubscribe", "punsubscribe", ] + + +class RedisPubSub(BasePubSub): + """ + A pubsub backend using Redis. + + You must supply a REDIS_URL setting that is passed through urlparse. + + Examples: + + * redis://localhost:6379/7 + * redis://rediscloud:asdfkjaslkdjflasdf@pub-redis-12345.us-east-1-1.2.ec2.garantiadata.com:12345 + """ + + required_settings = [ + { + "name": "REDIS_URL", + "obtain_at": """You must supply a REDIS_URL setting that is passed through urlparse. + +Examples: + +* redis://localhost:6379/7 +* redis://rediscloud:asdfkjaslkdjflasdf@pub-redis-12345.us-east-1-1.2.ec2.garantiadata.com:12345""", + }, + ] + + def __init__(self, settings, *args, **kwargs): + self.verify_settings(quiet=True) + super(RedisPubSub, self).__init__(*args, **kwargs) + url = parse.urlparse(settings.REDIS_URL) + + if hasattr(url, "path"): + db = url.path[1:] + else: + db = 0 + max_connections = int(getattr(settings, 'REDIS_MAX_CONNECTIONS', None)) + connection_pool = redis.ConnectionPool( + max_connections=max_connections, host=url.hostname, + port=url.port, db=db, password=url.password + ) + self.redis = redis.Redis(connection_pool=connection_pool) + self._pubsub = self.redis.pubsub() + + def publish_to_backend(self, topic, body_str): + logging.debug("publishing %s" % (topic,)) + return self.redis.publish(topic, body_str) + + def do_subscribe(self, topic): + logging.debug("subscribed to %s" % topic) + return self._pubsub.psubscribe(topic) + + def unsubscribe(self, topic): + return self._pubsub.punsubscribe(topic) + + def get_from_backend(self): + m = self._pubsub.get_message() + if m and m["type"] not in SKIP_TYPES: + return m + return None + + +def bootstrap(settings): + return RedisPubSub(settings) diff --git a/will/backends/pubsub/zeromq_pubsub.py b/will/backends/pubsub/zeromq_pubsub.py new file mode 100644 index 00000000..c1f10227 --- /dev/null +++ b/will/backends/pubsub/zeromq_pubsub.py @@ -0,0 +1,95 @@ +import logging +import traceback +import zmq +from .base import BasePubSub + +SKIP_TYPES = ["psubscribe", "punsubscribe", ] + +DIVIDER = "|WILL-SPLIT|" + + +class ZeroMQPubSub(BasePubSub): + """ + A pubsub backend using ZeroMQ. + + You must supply a ZEROMQ_URL setting that is passed directly to zmq.connect() + + Examples: + + * zeromq://localhost:63797 + * zeromq://ZeroMQ:asdfkjaslkdjflasdf@pub-zeromq-12345.us-east-1-1.2.ec2.zeromq.com:12345 + """ + + required_settings = [ + { + "name": "ZEROMQ_URL", + "obtain_at": """You must supply a ZEROMQ_URL setting that is passed through urlparse. + +Examples: + +* zeromq://localhost:63797 +* zeromq://ZeroMQ:asdfkjaslkdjflasdf@pub-zeromq-12345.us-east-1-1.2.ec2.zeromq.com:12345""", + }, + ] + + def __init__(self, settings, *args, **kwargs): + self.verify_settings(quiet=True) + logging.error( + "The ZeroMQ Backend isn't ready for prime-time yet. Please " + "test closely, and report any problems at Will's github page!" + ) + super(ZeroMQPubSub, self).__init__(*args, **kwargs) + context = zmq.Context.instance() + self.pub_socket = context.socket(zmq.PUB) + try: + self.pub_socket.bind(settings.ZEROMQ_URL) + except: + self.pub_socket.connect(settings.ZEROMQ_URL) + + sub_context = zmq.Context.instance() + self.sub_socket = sub_context.socket(zmq.SUB) + + self.sub_socket.connect(settings.ZEROMQ_URL) + self.sub_socket.setsockopt(zmq.SUBSCRIBE, '') + + # self.poller = zmq.Poller() + # self.poller.register(self.sub_socket, zmq.POLLIN) + + def publish_to_backend(self, topic, body_str): + return self.pub_socket.send("%s%s%s" % (topic, DIVIDER, body_str)) + + def do_subscribe(self, topic): + if type(topic) == type([]): + for t in topic: + self.sub_socket.setsockopt(zmq.SUBSCRIBE, t) + else: + self.sub_socket.setsockopt(zmq.SUBSCRIBE, topic) + + def unsubscribe(self, topic): + if type(topic) == type([]): + for t in topic: + self.sub_socket.setsockopt(zmq.UNSUBSCRIBE, t) + else: + self.sub_socket.setsockopt(zmq.UNSUBSCRIBE, topic) + + def get_from_backend(self): + try: + s = self.sub_socket.recv(zmq.DONTWAIT) + if s: + topic, m = s.split(DIVIDER) + if m and m["type"] not in SKIP_TYPES: + return m + except zmq.Again: + return None + + except (KeyboardInterrupt, SystemExit): + pass + except: + logging.critical( + "Error getting message from ZeroMQ backend: \n%s" % traceback.format_exc() + ) + return None + + +def bootstrap(settings): + return ZeroMQPubSub(settings) diff --git a/will/backends/storage/__init__.py b/will/backends/storage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/will/backends/storage/base.py b/will/backends/storage/base.py new file mode 100644 index 00000000..d4cbc07b --- /dev/null +++ b/will/backends/storage/base.py @@ -0,0 +1,41 @@ +import logging +import redis +from six.moves.urllib.parse import urlparse +from will.mixins import SettingsMixin, EncryptionMixin + + +class PrivateBaseStorageBackend(SettingsMixin, EncryptionMixin, object): + required_settings = [] + + def save(self, key, value, *args, **kwargs): + self.do_save(key, self.encrypt(value), *args, **kwargs) + + def load(self, key, *args, **kwargs): + try: + return self.decrypt(self.do_load(key, *args, **kwargs)) + except: + logging.warn("Error decrypting. Attempting unencrypted load for %s to ease migration." % key) + return self.do_load(key, *args, **kwargs) + + +class BaseStorageBackend(PrivateBaseStorageBackend): + """ + The base storage backend. All storage backends must supply the following methods: + __init__() - sets up the connection + do_save() - saves a single value to a key + do_load() - gets a value from the backend + clear() - deletes a key + clear_all_keys() - clears the db + """ + + def do_save(self, key, value, expire=None): + raise NotImplemented + + def do_load(self, key): + raise NotImplemented + + def clear(self, key): + raise NotImplemented + + def clear_all_keys(self): + raise NotImplemented diff --git a/will/backends/storage/couchbase_backend.py b/will/backends/storage/couchbase_backend.py new file mode 100644 index 00000000..2e9ac955 --- /dev/null +++ b/will/backends/storage/couchbase_backend.py @@ -0,0 +1,80 @@ +from six.moves.urllib import parse + +from couchbase import Couchbase, exceptions as cb_exc +from .base import BaseStorageBackend + + +class CouchbaseStorage(BaseStorageBackend): + """ + A storage backend using Couchbase + + You must supply a COUCHBASE_URL setting that is passed through urlparse. + All parameters supplied get passed through to Couchbase + + Examples: + + * couchbase:///bucket + * couchbase://hostname/bucket + * couchbase://host1,host2/bucket + * couchbase://hostname/bucket?password=123abc&timeout=5 + """ + + required_settings = [ + { + "name": "COUCHBASE_URL", + "obtain_at": """You must supply a COUCHBASE_URL setting that is passed through urlparse. +All parameters supplied get passed through to Couchbase + +Examples: + +* couchbase:///bucket +* couchbase://hostname/bucket +* couchbase://host1,host2/bucket +* couchbase://hostname/bucket?password=123abc&timeout=55""", + }, + ] + + def __init__(self, settings): + self.verify_settings(quiet=True) + url = parse.urlparse(settings.COUCHBASE_URL) + params = dict([ + param.split('=') + for param in url.query.split('&') + ]) + self.couchbase = Couchbase(host=url.hostname.split(','), + bucket=url.path.strip('/'), + port=url.port or 8091, + **params) + + def do_save(self, key, value, expire=None): + res = self.couchbase.set(key, value, ttl=expire) + return res.success + + def clear(self, key): + res = self.couchbase.delete(key) + return res.success + + def clear_all_keys(self): + """ + Couchbase doesn't support clearing all keys (flushing) without the + Admin username and password. It's not appropriate for Will to have + this information so we don't support clear_all_keys for CB. + """ + return "Sorry, you must flush the Couchbase bucket from the Admin UI" + + def do_load(self, key): + try: + res = self.couchbase.get(key) + return res.value + except cb_exc.NotFoundError: + pass + + def size(self): + """ + Couchbase doesn't support getting the size of the DB + """ + return "Unknown (See Couchbase Admin UI)" + + +def bootstrap(settings): + return CouchbaseStorage(settings) diff --git a/will/backends/storage/file_backend.py b/will/backends/storage/file_backend.py new file mode 100644 index 00000000..f9fe7ed9 --- /dev/null +++ b/will/backends/storage/file_backend.py @@ -0,0 +1,121 @@ +import logging +import os +import time + +from will.utils import sizeof_fmt +from .base import BaseStorageBackend + + +class FileStorageException(): + """ + A condition that should not occur happened in the FileStorage module + """ + pass + + +class FileStorage(BaseStorageBackend): + required_settings = [ + { + "name": "FILE_DIR", + "obtain_at": """You must supply a FILE_DIR setting that is a path to a directory. + +Examples: + + * /var/run/will/settings/ + * ~will/settings/""", + }, + ] + + """ + A storage backend using a local filesystem directory. + + Each setting is its own file. + + You must supply a FILE_DIR setting that is a path to a directory. + + Examples: + + * /var/run/will/settings/ + * ~will/settings/ + """ + def __init__(self, settings): + self.verify_settings(quiet=True) + self.dirname = os.path.abspath(os.path.expanduser(settings.FILE_DIR)) + self.dotfile = os.path.join(self.dirname, ".will_settings") + logging.debug("Using %s for local setting storage", self.dirname) + + if not os.path.exists(self.dirname): + # the directory doesn't exist, try to create it + os.makedirs(self.dirname, mode=0o700) + elif not os.path.exists(self.dotfile): + # the directory exists, but doesn't have our dot file in it + # if it has any other files in it then we bail out since we want to + # have full control over wiping out the contents of the directory + if len(self._all_setting_files()) > 0: + raise FileStorageException("%s is not empty, " + "will needs an empty directory for " + "settings" % (self.dirname,)) + + # update our dir & dotfile + os.chmod(self.dirname, 0o700) + with open(self.dotfile, 'a'): + os.utime(self.dotfile, None) + + def _all_setting_files(self): + return [ + os.path.join(self.dirname, f) + for f in os.listdir(self.dirname) + if os.path.isfile(os.path.join(self.dirname, f)) + ] + + def _key_paths(self, key): + key_path = os.path.join(self.dirname, key) + expire_path = os.path.join(self.dirname, '.' + key + '.expires') + return key_path, expire_path + + def do_save(self, key, value, expire=None): + key_path, expire_path = self._key_paths(key) + with open(key_path, 'w') as f: + f.write(value) + + if expire is not None: + with open(expire_path, 'w') as f: + f.write(str(int(time.time() + expire))) + elif os.path.exists(expire_path): + os.unlink(expire_path) + + def clear(self, key): + key_path, expire_path = self._key_paths(key) + if os.path.exists(key_path): + os.unlink(key_path) + if os.path.exists(expire_path): + os.unlink(expire_path) + + def clear_all_keys(self): + for filename in self._all_setting_files(): + os.unlink(filename) + + def do_load(self, key): + key_path, expire_path = self._key_paths(key) + + if os.path.exists(expire_path): + with open(expire_path, 'r') as f: + expire_at = f.read() + if time.time() > int(expire_at): + # the current value has expired + self.clear(key) + return + + if os.path.exists(key_path): + with open(key_path, 'r') as f: + return f.read() + + def size(self): + return sizeof_fmt(sum([ + os.path.getsize(filename) + for filename in self._all_setting_files() + ])) + + +def bootstrap(settings): + return FileStorage(settings) diff --git a/will/backends/storage/redis_backend.py b/will/backends/storage/redis_backend.py new file mode 100644 index 00000000..7fed7359 --- /dev/null +++ b/will/backends/storage/redis_backend.py @@ -0,0 +1,61 @@ +import redis +from six.moves.urllib import parse +from .base import BaseStorageBackend + + +class RedisStorage(BaseStorageBackend): + required_settings = [ + { + "name": "REDIS_URL", + "obtain_at": """You must supply a REDIS_URL setting that is passed through urlparse. + +Examples: + +* redis://localhost:6379/7 +* redis://rediscloud:asdfkjaslkdjflasdf@pub-redis-12345.us-east-1-1.2.ec2.garantiadata.com:12345""", + }, + ] + + """ + A storage backend using Redis. + + You must supply a REDIS_URL setting that is passed through urlparse. + + Examples: + + * redis://localhost:6379/7 + * redis://rediscloud:asdfkjaslkdjflasdf@pub-redis-12345.us-east-1-1.2.ec2.garantiadata.com:12345 + """ + def __init__(self, settings): + self.verify_settings(quiet=True) + url = parse.urlparse(settings.REDIS_URL) + + if hasattr(url, "path"): + db = url.path[1:] + else: + db = 0 + max_connections = int(getattr(settings, 'REDIS_MAX_CONNECTIONS', None)) + connection_pool = redis.ConnectionPool( + max_connections=max_connections, host=url.hostname, + port=url.port, db=db, password=url.password + ) + self.redis = redis.Redis(connection_pool=connection_pool) + + def do_save(self, key, value, expire=None): + return self.redis.set(key, value, ex=expire) + + def clear(self, key): + return self.redis.delete(key) + + def clear_all_keys(self): + return self.redis.flushdb() + + def do_load(self, key): + return self.redis.get(key) + + def size(self): + return self.redis.info()["used_memory_human"] + + +def bootstrap(settings): + return RedisStorage(settings) diff --git a/will/decorators.py b/will/decorators.py index 994f6656..12c16071 100644 --- a/will/decorators.py +++ b/will/decorators.py @@ -1,11 +1,29 @@ + +def deprecation_warning_for_admin(f): + err = ( + "admin_only=True is deprecated and is being used by the `%s` method.\n" % (f.__name__, ) + + " Please use ACLs instead. admin_only will be removed at the end of 2017." + ) + return err + + +def passthrough_decorator(*args, **kwargs): + def wrap(f): + def wrapped_f(*args, **kwargs): + return f(*args, **kwargs) + return wrapped_f + return wrap + + def respond_to(regex, include_me=False, case_sensitive=False, multiline=False, admin_only=False, acl=set()): def wrap(f): passed_args = [] + if admin_only: + f.warnings = deprecation_warning_for_admin(f) def wrapped_f(*args, **kwargs): f(*args, **kwargs) wrapped_f.will_fn_metadata = getattr(f, "will_fn_metadata", {}) - wrapped_f.will_fn_metadata["listener_regex"] = regex wrapped_f.will_fn_metadata["case_sensitive"] = case_sensitive wrapped_f.will_fn_metadata["multiline"] = multiline @@ -16,6 +34,9 @@ def wrapped_f(*args, **kwargs): wrapped_f.will_fn_metadata["listener_args"] = passed_args wrapped_f.will_fn_metadata["__doc__"] = f.__doc__ wrapped_f.will_fn_metadata["listeners_acl"] = acl + if getattr(f, "warnings", None): + wrapped_f.will_fn_metadata["warnings"] = getattr(f, "warnings") + return wrapped_f return wrap @@ -30,6 +51,8 @@ def wrapped_f(*args, **kwargs): wrapped_f.will_fn_metadata["function_name"] = f.__name__ wrapped_f.will_fn_metadata["sched_args"] = sched_args wrapped_f.will_fn_metadata["sched_kwargs"] = sched_kwargs + if getattr(f, "warnings", None): + wrapped_f.will_fn_metadata["warnings"] = getattr(f, "warnings") return wrapped_f return wrap @@ -37,6 +60,8 @@ def wrapped_f(*args, **kwargs): def hear(regex, include_me=False, case_sensitive=False, multiline=False, admin_only=False, acl=set()): def wrap(f): passed_args = [] + if admin_only: + f.warnings = deprecation_warning_for_admin(f) def wrapped_f(*args, **kwargs): f(*args, **kwargs) @@ -51,6 +76,8 @@ def wrapped_f(*args, **kwargs): wrapped_f.will_fn_metadata["listener_args"] = passed_args wrapped_f.will_fn_metadata["__doc__"] = f.__doc__ wrapped_f.will_fn_metadata["listeners_acl"] = acl + if getattr(f, "warnings", None): + wrapped_f.will_fn_metadata["warnings"] = getattr(f, "warnings") return wrapped_f @@ -68,6 +95,8 @@ def wrapped_f(*args, **kwargs): wrapped_f.will_fn_metadata["end_hour"] = int(end_hour) wrapped_f.will_fn_metadata["day_of_week"] = day_of_week wrapped_f.will_fn_metadata["num_times_per_day"] = int(num_times_per_day) + if getattr(f, "warnings", None): + wrapped_f.will_fn_metadata["warnings"] = getattr(f, "warnings") return wrapped_f return wrap @@ -81,7 +110,7 @@ def rendered_template(template_name, context=None, custom_filters=[]): loader = FileSystemLoader(template_dirs) env = Environment(loader=loader) - if isinstance(custom_filters, list()): + if isinstance(custom_filters, list): for custom_filter in custom_filters: env.filters[custom_filter.__name__] = custom_filter @@ -92,7 +121,7 @@ def rendered_template(template_name, context=None, custom_filters=[]): def wrap(f): def wrapped_f(*args, **kwargs): context = f(*args, **kwargs) - if isinstance(context, dict()): + if isinstance(context, dict): template = env.get_template(template_name) return template.render(**context) else: diff --git a/will/listener.py b/will/listener.py deleted file mode 100644 index 327f310f..00000000 --- a/will/listener.py +++ /dev/null @@ -1,193 +0,0 @@ -import logging -import re -import threading -import traceback -from sleekxmpp import ClientXMPP - -import settings -from utils import Bunch -from mixins import RosterMixin, RoomMixin, HipChatMixin - - -class WillXMPPClientMixin(ClientXMPP, RosterMixin, RoomMixin, HipChatMixin): - - def start_xmpp_client(self): - logger = logging.getLogger(__name__) - ClientXMPP.__init__(self, "%s/bot" % settings.USERNAME, settings.PASSWORD) - - if settings.USE_PROXY: - self.use_proxy = True - self.proxy_config = { - 'host': settings.PROXY_HOSTNAME, - 'port': settings.PROXY_PORT, - 'username': settings.PROXY_USERNAME, - 'password': settings.PROXY_PASSWORD, - } - - self.rooms = [] - self.default_room = settings.DEFAULT_ROOM - - # Property boostraps the list - self.available_rooms - for r in settings.ROOMS: - if r != "": - if not hasattr(self, "default_room"): - self.default_room = r - - try: - self.rooms.append(self.available_rooms[r]) - except KeyError: - logger.error( - u'"{0}" is not an available room, ask' - ' "@{1} what are the rooms?" for the full list.' - .format(r, settings.HANDLE)) - - self.nick = settings.NAME - self.handle = settings.HANDLE - self.handle_regex = re.compile("@%s" % self.handle) - - self.whitespace_keepalive = True - self.whitespace_keepalive_interval = 30 - - if settings.ALLOW_INSECURE_HIPCHAT_SERVER is True: - self.add_event_handler('ssl_invalid_cert', lambda cert: True) - - self.add_event_handler("roster_update", self.join_rooms) - self.add_event_handler("session_start", self.session_start) - self.add_event_handler("message", self.message_recieved) - self.add_event_handler("groupchat_message", self.room_message) - - self.register_plugin('xep_0045') # MUC - - def session_start(self, event): - self.send_presence() - self.get_roster() - - def join_rooms(self, event): - self.update_will_roster_and_rooms() - - for r in self.rooms: - if "xmpp_jid" in r: - self.plugin['xep_0045'].joinMUC(r["xmpp_jid"], self.nick, wait=True) - - def update_will_roster_and_rooms(self): - internal_roster = self.load('will_roster', {}) - # Loop through the connected rooms - for roster_id in self.roster: - - cur_roster = self.roster[roster_id] - # Loop through the users in a given room - for user_id in cur_roster: - user_data = cur_roster[user_id] - if user_data["name"] != "": - # If we don't have this user in the internal_roster, add them. - if not user_id in internal_roster: - internal_roster[user_id] = Bunch() - - hipchat_id = user_id.split("@")[0].split("_")[1] - # Update their info - internal_roster[user_id].update({ - "name": user_data["name"], - "jid": user_id, - "hipchat_id": hipchat_id, - }) - - # If we don't have a nick yet, pull it and mention_name off the master user list. - if not hasattr(internal_roster[user_id], "nick"): - user_data = self.full_hipchat_user_list[hipchat_id] - internal_roster[user_id].nick = user_data["mention_name"] - internal_roster[user_id].mention_name = user_data["mention_name"] - - # If it's me, save that info! - if internal_roster[user_id]["name"] == self.nick: - self.me = internal_roster[user_id] - - self.save("will_roster", internal_roster) - - self.update_available_rooms() - - def room_message(self, msg): - self._handle_message_listeners(msg) - - def message_recieved(self, msg): - if msg['type'] in ('chat', 'normal'): - self._handle_message_listeners(msg) - - def real_sender_jid(self, msg): - # There's a bug in sleekXMPP where it doesn't set the "from_jid" properly. - # Thus, this hideous hack. - msg_str = "%s" % msg - start = 'from_jid="' - start_pos = msg_str.find(start) - if start_pos != -1: - cut_start = start_pos + len(start) - return msg_str[cut_start:msg_str.find('"', cut_start)] - - return msg["from"] - - def _handle_message_listeners(self, msg): - if ( - # I've been asked to listen to my own messages - self.some_listeners_include_me - # or we're in a 1 on 1 chat and I didn't send it - or (msg['type'] in ('chat', 'normal') and self.real_sender_jid(msg) != self.me.jid) - # we're in group chat and I didn't send it - or (msg["type"] == "groupchat" and msg['mucnick'] != self.nick) - ): - body = msg["body"].strip() - - sent_directly_to_me = False - # If it's sent directly to me, strip off "@will" from the start. - if body[:len(self.handle) + 1].lower() == ("@%s" % self.handle).lower(): - body = body[len(self.handle) + 1:].strip() - msg["body"] = body - - sent_directly_to_me = True - - # Make the message object a bit friendlier - msg.room = self.get_room_from_message(msg) - msg.sender = self.get_user_from_message(msg) - - for l in self.message_listeners: - search_matches = l["regex"].search(body) - if ( - search_matches # The search regex matches and - # It's not from me, or this search includes me, and - and (msg['mucnick'] != self.nick or l["include_me"]) - # I'm mentioned, or this is an overheard, or we're in a 1-1 - and (msg['type'] in ('chat', 'normal') or not l["direct_mentions_only"] or - self.handle_regex.search(body) or sent_directly_to_me) - # It's from admins only and sender is an admin, or it's not from admins only - and ((l['admin_only'] and self.message_is_from_admin(msg)) or (not l['admin_only'])) - # It's available only to the members of one or more ACLs, or no ACL in use - and ((len(l['acl']) > 0 and self.message_is_allowed(msg, l['acl'])) or (len(l['acl']) == 0)) - ): - try: - thread_args = [msg, ] + l["args"] - - def fn(listener, args, kwargs): - try: - listener["fn"](*args, **kwargs) - except: - content = "I ran into trouble running %s.%s:\n\n%s" % ( - listener["class_name"], - listener["function_name"], - traceback.format_exc(), - ) - - if msg is None or msg["type"] == "groupchat": - if msg.sender and "nick" in msg.sender: - content = "@%s %s" % (msg.sender["nick"], content) - self.send_room_message(msg.room["room_id"], content, color="red") - elif msg['type'] in ('chat', 'normal'): - self.send_direct_message(msg.sender["hipchat_id"], content) - - thread = threading.Thread(target=fn, args=(l, thread_args, search_matches.groupdict())) - thread.start() - except: - logging.critical( - "Error running %s. \n\n%s\nContinuing...\n" % ( - l["function_name"], - traceback.format_exc() - ) - ) diff --git a/will/main.py b/will/main.py index e6187aa6..514bc356 100644 --- a/will/main.py +++ b/will/main.py @@ -1,31 +1,42 @@ # -*- coding: utf-8 -*- -import logging -import inspect +import copy +import datetime import imp -import os +from importlib import import_module +import inspect +import logging +from multiprocessing import Process, Queue import operator +import os +from os.path import abspath, dirname import re +import signal import sys +import threading import time import traceback -from clint.textui import colored, puts, indent -from os.path import abspath, dirname -from multiprocessing import Process, Queue +try: + from yappi import profile as yappi_profile +except: + from will.decorators import passthrough_decorator as yappi_profile +from clint.textui import colored, puts, indent import bottle -from listener import WillXMPPClientMixin -from mixins import ScheduleMixin, StorageMixin, ErrorMixin, HipChatMixin,\ - RoomMixin, PluginModulesLibraryMixin, EmailMixin -from scheduler import Scheduler -import settings -from utils import show_valid, error, warn, print_head + +from will import settings +from will.backends import analysis, execution, generation, io_adapters +from will.backends.io_adapters.base import Event +from will.mixins import ScheduleMixin, StorageMixin, ErrorMixin, SleepMixin,\ + PluginModulesLibraryMixin, EmailMixin, PubSubMixin +from will.scheduler import Scheduler +from will.utils import show_valid, show_invalid, error, warn, note, print_head, Bunch # Force UTF8 if sys.version_info < (3, 0): - reload(sys) + reload(sys) # flake8: noqa sys.setdefaultencoding('utf8') else: raw_input = input @@ -40,8 +51,20 @@ sys.path.append(os.path.join(PROJECT_ROOT, "will")) -class WillBot(EmailMixin, WillXMPPClientMixin, StorageMixin, ScheduleMixin, - ErrorMixin, RoomMixin, HipChatMixin, PluginModulesLibraryMixin): +def yappi_aggregate(func, stats): + if hasattr(settings, "PROFILING_ENABLED") and settings.PROFILING_ENABLED: + fname = "callgrind.%s" % (func.__name__) + + try: + stats.add(fname) + except IOError: + pass + + stats.save("will_profiles/%s" % fname, "callgrind") + + +class WillBot(EmailMixin, StorageMixin, ScheduleMixin, PubSubMixin, SleepMixin, + ErrorMixin, PluginModulesLibraryMixin): def __init__(self, **kwargs): if "template_dirs" in kwargs: @@ -52,12 +75,19 @@ def __init__(self, **kwargs): log_level = getattr(settings, 'LOGLEVEL', logging.ERROR) logging.basicConfig( level=log_level, - format='%(levelname)-8s %(message)s' + format='%(asctime)s [%(levelname)s] %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S', ) + # Bootstrap exit code. + self.exiting = False # Find all the PLUGINS modules - plugins = settings.PLUGINS - self.plugins_dirs = {} + try: + plugins = settings.PLUGINS + self.plugins_dirs = {} + except: + # We're missing settings. They handle that. + sys.exit(1) # Set template dirs. full_path_template_dirs = [] @@ -95,57 +125,84 @@ def __init__(self, **kwargs): os.environ["WILL_TEMPLATE_DIRS_PICKLED"] =\ ";;".join(full_path_template_dirs) + @yappi_profile(return_callback=yappi_aggregate) def bootstrap(self): print_head() - self.verify_environment() self.load_config() - self.verify_rooms() self.bootstrap_storage_mixin() + self.bootstrap_pubsub_mixin() self.bootstrap_plugins() self.verify_plugin_settings() + started = self.verify_io() + if started: + puts("Bootstrapping complete.") - puts("Bootstrapping complete.") - puts("\nStarting core processes:") - # Scheduler - scheduler_thread = Process(target=self.bootstrap_scheduler) - # scheduler_thread.daemon = True + # Save help modules. + self.save("help_modules", self.help_modules) - # Bottle - bottle_thread = Process(target=self.bootstrap_bottle) - # bottle_thread.daemon = True + puts("\nStarting core processes:") - # XMPP Listener - xmpp_thread = Process(target=self.bootstrap_xmpp) - # xmpp_thread.daemon = True + # try: + # Exit handlers. + # signal.signal(signal.SIGINT, self.handle_sys_exit) + # # TODO this hangs for some reason. + # signal.signal(signal.SIGTERM, self.handle_sys_exit) - with indent(2): - try: - # Start up threads. - xmpp_thread.start() - scheduler_thread.start() - bottle_thread.start() - errors = self.get_startup_errors() - if len(errors) > 0: - default_room = self.get_room_from_name_or_id(settings.DEFAULT_ROOM)["room_id"] - error_message = "FYI, I ran into some problems while starting up:" - for err in errors: - error_message += "\n%s\n" % err - self.send_room_message(default_room, error_message, color="yellow") - puts(colored.red(error_message)) - - while True: - time.sleep(100) - except (KeyboardInterrupt, SystemExit): - scheduler_thread.terminate() - bottle_thread.terminate() - xmpp_thread.terminate() - print '\n\nReceived keyboard interrupt, quitting threads.', - while (scheduler_thread.is_alive() or - bottle_thread.is_alive() or - xmpp_thread.is_alive()): - sys.stdout.write(".") - sys.stdout.flush() - time.sleep(0.5) + # Scheduler + self.scheduler_thread = Process(target=self.bootstrap_scheduler) + + # Bottle + self.bottle_thread = Process(target=self.bootstrap_bottle) + + # Event handler + self.incoming_event_thread = Process(target=self.bootstrap_event_handler) + + self.io_threads = [] + self.analysis_threads = [] + self.generation_threads = [] + + with indent(2): + try: + # Start up threads. + self.bootstrap_io() + self.bootstrap_analysis() + self.bootstrap_generation() + self.bootstrap_execution() + + self.scheduler_thread.start() + self.bottle_thread.start() + self.incoming_event_thread.start() + + errors = self.get_startup_errors() + if len(errors) > 0: + error_message = "FYI, I ran into some problems while starting up:" + for err in errors: + error_message += "\n%s\n" % err + puts(colored.red(error_message)) + + self.stdin_listener_thread = False + if self.has_stdin_io_backend: + + self.current_line = "" + while True: + for line in sys.stdin.readline(): + if "\n" in line: + self.publish( + "message.incoming.stdin", + Event( + type="message.incoming.stdin", + content=self.current_line, + ) + ) + self.current_line = "" + else: + self.current_line += line + self.sleep_for_event_loop(2) + else: + while True: + time.sleep(100) + except (KeyboardInterrupt, SystemExit): + self.handle_sys_exit() def verify_individual_setting(self, test_setting, quiet=False): if not test_setting.get("only_if", True): @@ -168,106 +225,244 @@ def verify_individual_setting(self, test_setting, quiet=False): """ % test_setting) return False - def verify_environment(self): + def load_config(self): + puts("Loading configuration...") + with indent(2): + settings.import_settings(quiet=False) + puts("") + + @yappi_profile(return_callback=yappi_aggregate) + def verify_io(self): + puts("Verifying IO backends...") missing_settings = False - required_settings = [ - { - "name": "WILL_USERNAME", - "obtain_at": """1. Go to hipchat, and create a new user for will. -2. Log into will, and go to Account settings>XMPP/Jabber Info. -3. On that page, the 'Jabber ID' is the value you want to use.""", - }, - { - "name": "WILL_PASSWORD", - "obtain_at": ( - "1. Go to hipchat, and create a new user for will. " - "Note that password - this is the value you want. " - "It's used for signing in via XMPP." - ), - }, - { - "name": "WILL_V2_TOKEN", - "obtain_at": """1. Log into hipchat using will's user. -2. Go to https://your-org.hipchat.com/account/api -3. Create a token. -4. Copy the value - this is the WILL_V2_TOKEN.""", - }, - { - "name": "WILL_REDIS_URL", - "only_if": getattr(settings, "STORAGE_BACKEND", "redis") == "redis", - "obtain_at": """1. Set up an accessible redis host locally or in production -2. Set WILL_REDIS_URL to its full value, i.e. redis://localhost:6379/7""", - }, - ] + missing_setting_error_messages = [] + one_valid_backend = False + self.valid_io_backends = [] - puts("") - puts("Verifying environment...") + if not hasattr(settings, "IO_BACKENDS"): + settings.IO_BACKENDS = ["will.backends.io_adapters.shell", ] + # Try to import them all, catch errors and output trouble if we hit it. + for b in settings.IO_BACKENDS: + with indent(2): + try: + path_name = None + for mod in b.split('.'): + if path_name is not None: + path_name = [path_name] + file_name, path_name, description = imp.find_module(mod, path_name) - for r in required_settings: - if not self.verify_individual_setting(r): - missing_settings = True + # show_valid("%s" % b) + module = import_module(b) + for class_name, cls in inspect.getmembers(module, predicate=inspect.isclass): + if ( + hasattr(cls, "is_will_iobackend") and + cls.is_will_iobackend and + class_name != "IOBackend" and + class_name != "StdInOutIOBackend" + ): + c = cls() + show_valid(c.friendly_name) + c.verify_settings() + one_valid_backend = True + self.valid_io_backends.append(b) + except EnvironmentError as e: + puts(colored.red(" ✗ %s is missing settings, and will be disabled." % b)) + puts() + + missing_settings = True - if missing_settings: + except Exception as e: + error_message = ( + "IO backend %s is missing. Please either remove it \nfrom config.py " + "or WILL_IO_BACKENDS, or provide it somehow (pip install, etc)." + ) % b + puts(colored.red("✗ %s" % b)) + puts() + puts(error_message) + puts() + puts(traceback.format_exc()) + missing_setting_error_messages.append(error_message) + missing_settings = True + + if missing_settings and not one_valid_backend: + puts("") error( - "Will was unable to start because some required environment " - "variables are missing. Please fix them and try again!" + "Unable to find a valid IO backend - will has no way to talk " + "or listen!\n Quitting now, please look at the above errors!\n" ) - sys.exit(1) - else: + self.handle_sys_exit() + return False + puts() + return True + + @yappi_profile(return_callback=yappi_aggregate) + def verify_analysis(self): + puts("Verifying Analysis backends...") + missing_settings = False + missing_setting_error_messages = [] + one_valid_backend = False + + if not hasattr(settings, "ANALYZE_BACKENDS"): + settings.ANALYZE_BACKENDS = ["will.backends.analysis.nothing", ] + # Try to import them all, catch errors and output trouble if we hit it. + for b in settings.ANALYZE_BACKENDS: + with indent(2): + try: + path_name = None + for mod in b.split('.'): + if path_name is not None: + path_name = [path_name] + file_name, path_name, description = imp.find_module(mod, path_name) + + one_valid_backend = True + show_valid("%s" % b) + except ImportError as e: + error_message = ( + "Analysis backend %s is missing. Please either remove it \nfrom config.py " + "or WILL_ANALYZE_BACKENDS, or provide it somehow (pip install, etc)." + ) % b + puts(colored.red("✗ %s" % b)) + puts() + puts(error_message) + puts() + puts(traceback.format_exc()) + missing_setting_error_messages.append(error_message) + missing_settings = True + + if missing_settings and not one_valid_backend: puts("") + error( + "Unable to find a valid IO backend - will has no way to talk " + "or listen!\n Quitting now, please look at the above errors!\n" + ) + sys.exit(1) + puts() - puts("Verifying credentials...") - # Parse 11111_222222@chat.hipchat.com into id, where 222222 is the id. - user_id = settings.USERNAME.split('@')[0].split('_')[1] + @yappi_profile(return_callback=yappi_aggregate) + def verify_generate(self): + puts("Verifying Generation backends...") + missing_settings = False + missing_setting_error_messages = [] + one_valid_backend = False - # Splitting into a thread. Necessary because *BSDs (including OSX) don't have threadsafe DNS. - # http://stackoverflow.com/questions/1212716/python-interpreter-blocks-multithreaded-dns-requests - q = Queue() - p = Process(target=self.get_hipchat_user, args=(user_id,), kwargs={"q": q, }) - p.start() - user_data = q.get() - p.join() + if not hasattr(settings, "GENERATION_BACKENDS"): + settings.GENERATION_BACKENDS = ["will.backends.generation.strict_regex", ] + # Try to import them all, catch errors and output trouble if we hit it. + for b in settings.GENERATION_BACKENDS: + with indent(2): + try: + path_name = None + for mod in b.split('.'): + if path_name is not None: + path_name = [path_name] + file_name, path_name, description = imp.find_module(mod, path_name) - if "error" in user_data: - error("We ran into trouble: '%(message)s'" % user_data["error"]) + one_valid_backend = True + show_valid("%s" % b) + except ImportError as e: + error_message = ( + "Generation backend %s is missing. Please either remove it \nfrom config.py " + "or WILL_GENERATION_BACKENDS, or provide it somehow (pip install, etc)." + ) % b + puts(colored.red("✗ %s" % b)) + puts() + puts(error_message) + puts() + puts(traceback.format_exc()) + missing_setting_error_messages.append(error_message) + missing_settings = True + + if missing_settings and not one_valid_backend: + puts("") + error( + "Unable to find a valid IO backend - will has no way to talk " + "or listen!\n Quitting now, please look at the above errors!\n" + ) sys.exit(1) - with indent(2): - show_valid("%s authenticated" % user_data["name"]) - os.environ["WILL_NAME"] = user_data["name"] - show_valid("@%s verified as handle" % user_data["mention_name"]) - os.environ["WILL_HANDLE"] = user_data["mention_name"] + puts() - puts("") + @yappi_profile(return_callback=yappi_aggregate) + def verify_execution(self): + puts("Verifying Execution backend...") + missing_settings = False + missing_setting_error_messages = [] + one_valid_backend = False - def load_config(self): - puts("Loading configuration...") - with indent(2): - settings.import_settings(quiet=False) - puts("") + if not hasattr(settings, "EXECUTION_BACKENDS"): + settings.EXECUTION_BACKENDS = ["will.backends.execution.all", ] - def verify_rooms(self): - puts("Verifying rooms...") - # If we're missing ROOMS, join all of them. with indent(2): - if settings.ROOMS is None: - # Yup. Thanks, BSDs. - q = Queue() - p = Process(target=self.update_available_rooms, args=(), kwargs={"q": q, }) - p.start() - rooms_list = q.get() - show_valid("Joining all %s known rooms." % len(rooms_list)) - os.environ["WILL_ROOMS"] = ";".join(rooms_list) - p.join() - settings.import_settings() - else: - show_valid( - "Joining the %s room%s specified." % ( - len(settings.ROOMS), - "s" if len(settings.ROOMS) > 1 else "" - ) - ) - puts("") + for b in settings.EXECUTION_BACKENDS: + try: + path_name = None + for mod in b.split('.'): + if path_name is not None: + path_name = [path_name] + file_name, path_name, description = imp.find_module(mod, path_name) + + one_valid_backend = True + show_valid("%s" % b) + except ImportError as e: + error_message = ( + "Execution backend %s is missing. Please either remove it \nfrom config.py " + "or WILL_EXECUTION_BACKENDS, or provide it somehow (pip install, etc)." + ) % b + puts(colored.red("✗ %s" % b)) + puts() + puts(error_message) + puts() + puts(traceback.format_exc()) + missing_setting_error_messages.append(error_message) + missing_settings = True + + if missing_settings and not one_valid_backend: + puts("") + error( + "Unable to find a valid IO backend - will has no way to talk " + "or listen!\n Quitting now, please look at the above errors!\n" + ) + sys.exit(1) + puts() + @yappi_profile(return_callback=yappi_aggregate) + def bootstrap_execution(self): + missing_setting_error_messages = [] + self.execution_backends = [] + self.running_execution_threads = [] + execution_backends = getattr(settings, "EXECUTION_BACKENDS", ["will.backends.execution.all", ]) + for b in execution_backends: + module = import_module(b) + for class_name, cls in inspect.getmembers(module, predicate=inspect.isclass): + try: + if ( + hasattr(cls, "is_will_execution_backend") and + cls.is_will_execution_backend and + class_name != "ExecutionBackend" + ): + c = cls(bot=self) + self.execution_backends.append(c) + show_valid("Execution: %s Backend started." % cls.__name__) + except ImportError as e: + error_message = ( + "Execution backend %s is missing. Please either remove it \nfrom config.py " + "or WILL_EXECUTION_BACKENDS, or provide it somehow (pip install, etc)." + ) % settings.EXECUTION_BACKENDS + puts(colored.red("✗ %s" % settings.EXECUTION_BACKENDS)) + puts() + puts(error_message) + puts() + puts(traceback.format_exc()) + missing_setting_error_messages.append(error_message) + + if len(self.execution_backends) == 0: + puts("") + error( + "Unable to find a valid execution backend - will has no way to make decisions!" + "\n Quitting now, please look at the above error!\n" + ) + sys.exit(1) + + @yappi_profile(return_callback=yappi_aggregate) def verify_plugin_settings(self): puts("Verifying settings requested by plugins...") @@ -297,23 +492,241 @@ def verify_plugin_settings(self): else: puts("") + def handle_sys_exit(self, *args, **kwargs): + # if not self.exiting: + try: + sys.stdout.write("\n\nReceived shutdown, quitting threads.") + sys.stdout.flush() + self.exiting = True + + if "WILL_EPHEMERAL_SECRET_KEY" in os.environ: + os.environ["WILL_SECRET_KEY"] = "" + os.environ["WILL_EPHEMERAL_SECRET_KEY"] = "" + + if hasattr(self, "scheduler_thread") and self.scheduler_thread: + try: + self.scheduler_thread.terminate() + except KeyboardInterrupt: + pass + if hasattr(self, "bottle_thread") and self.bottle_thread: + try: + self.bottle_thread.terminate() + except KeyboardInterrupt: + pass + if hasattr(self, "incoming_event_thread") and self.incoming_event_thread: + try: + self.incoming_event_thread.terminate() + except KeyboardInterrupt: + pass + + # if self.stdin_listener_thread: + # self.stdin_listener_thread.terminate() + + self.publish("system.terminate", {}) + + if hasattr(self, "analysis_threads") and self.analysis_threads: + for t in self.analysis_threads: + try: + t.terminate() + except KeyboardInterrupt: + pass + + if hasattr(self, "generation_threads") and self.generation_threads: + for t in self.generation_threads: + try: + t.terminate() + except KeyboardInterrupt: + pass + + if hasattr(self, "running_execution_threads") and self.running_execution_threads: + for t in self.running_execution_threads: + try: + t.terminate() + except KeyboardInterrupt: + pass + except: + print("\n\n\nException while exiting!!") + import traceback + traceback.print_exc() + sys.exit(1) + + while ( + (hasattr(self, "scheduler_thread") and self.scheduler_thread and self.scheduler_thread and self.scheduler_thread.is_alive()) or + (hasattr(self, "scheduler_thread") and self.scheduler_thread and self.bottle_thread and self.bottle_thread.is_alive()) or + (hasattr(self, "scheduler_thread") and self.scheduler_thread and self.incoming_event_thread and self.incoming_event_thread.is_alive()) or + # self.stdin_listener_thread.is_alive() or + any([t.is_alive() for t in self.io_threads]) or + any([t.is_alive() for t in self.analysis_threads]) or + any([t.is_alive() for t in self.generation_threads]) or + any([t.is_alive() for t in self.running_execution_threads]) + # or + # ("hipchat" in settings.CHAT_BACKENDS and xmpp_thread and xmpp_thread.is_alive()) + ): + sys.stdout.write(".") + sys.stdout.flush() + time.sleep(0.5) + print(". done.\n") + sys.exit(1) + + @yappi_profile(return_callback=yappi_aggregate) + def bootstrap_event_handler(self): + self.analysis_timeout = getattr(settings, "ANALYSIS_TIMEOUT_MS", 2000) + self.generation_timeout = getattr(settings, "GENERATION_TIMEOUT_MS", 2000) + self.pubsub.subscribe(["message.*", "analysis.*", "generation.*"]) + + # TODO: change this to the number of running analysis threads + num_analysis_threads = len(settings.ANALYZE_BACKENDS) + num_generation_threads = len(settings.GENERATION_BACKENDS) + analysis_threads = {} + generation_threads = {} + + while True: + try: + event = self.pubsub.get_message() + if event and hasattr(event, "type"): + now = datetime.datetime.now() + logging.info("%s - %s" % (event.type, event.original_incoming_event_hash)) + logging.debug("\n\n *** Event (%s): %s\n\n" % (event.type, event)) + + # TODO: Order by most common. + if event.type == "message.incoming": + # A message just got dropped off one of the IO Backends. + # Send it to analysis. + + analysis_threads[event.original_incoming_event_hash] = { + "count": 0, + "timeout_end": now + datetime.timedelta(seconds=self.analysis_timeout / 1000), + "original_incoming_event": event, + "working_event": event, + } + self.pubsub.publish("analysis.start", event.data.original_incoming_event, reference_message=event) + + elif event.type == "analysis.complete": + q = analysis_threads[event.original_incoming_event_hash] + q["working_event"].update({"analysis": event.data}) + q["count"] += 1 + logging.info("Analysis for %s: %s/%s" % (event.original_incoming_event_hash, q["count"], num_analysis_threads)) + + if q["count"] >= num_analysis_threads or now > q["timeout_end"]: + # done, move on. + generation_threads[event.original_incoming_event_hash] = { + "count": 0, + "timeout_end": ( + now + + datetime.timedelta(seconds=self.generation_timeout / 1000) + ), + "original_incoming_event": q["original_incoming_event"], + "working_event": q["working_event"], + } + try: + del analysis_threads[event.original_incoming_event_hash] + except: + pass + self.pubsub.publish("generation.start", q["working_event"], reference_message=q["original_incoming_event"]) + + elif event.type == "generation.complete": + q = generation_threads[event.original_incoming_event_hash] + if not hasattr(q["working_event"], "generation_options"): + q["working_event"].generation_options = [] + if hasattr(event, "data") and len(event.data) > 0: + for d in event.data: + q["working_event"].generation_options.append(d) + q["count"] += 1 + logging.info("Generation for %s: %s/%s" % (event.original_incoming_event_hash, q["count"], num_generation_threads)) + + if q["count"] >= num_generation_threads or now > q["timeout_end"]: + # done, move on to execution. + for b in self.execution_backends: + try: + logging.info("Executing for %s on %s" % (b, event.original_incoming_event_hash)) + b.handle_execution(q["working_event"]) + except: + logging.critical( + "Error running %s for %s. \n\n%s\nContinuing...\n" % ( + b, + event.original_incoming_event_hash, + traceback.format_exc() + ) + ) + break + try: + del generation_threads[event.original_incoming_event_hash] + except: + pass + + elif event.type == "message.no_response": + logging.info("Publishing no response for %s" % (event.original_incoming_event_hash,)) + logging.info(event.data.__dict__) + try: + self.publish("message.outgoing.%s" % event.data.backend, event) + except: + logging.critical( + "Error publishing no_response for %s. \n\n%s\nContinuing...\n" % ( + event.original_incoming_event_hash, + traceback.format_exc() + ) + ) + pass + elif event.type == "message.not_allowed": + logging.info("Publishing not allowed for %s" % (event.original_incoming_event_hash,)) + try: + self.publish("message.outgoing.%s" % event.data.backend, event) + except: + logging.critical( + "Error publishing not_allowed for %s. \n\n%s\nContinuing...\n" % ( + event.original_incoming_event_hash, + traceback.format_exc() + ) + ) + pass + else: + self.sleep_for_event_loop() + # except KeyError: + # pass + except: + logging.exception("Error handling message") + + @yappi_profile(return_callback=yappi_aggregate) def bootstrap_storage_mixin(self): puts("Bootstrapping storage...") try: self.bootstrap_storage() + # Make sure settings are there. + self.storage.verify_settings() with indent(2): show_valid("Bootstrapped!") puts("") - except ImportError, e: - module_name = traceback.format_exc(e).split(" ")[-1] + except ImportError : + module_name = traceback.format_exc().split(" ")[-1] error("Unable to bootstrap storage - attempting to load %s" % module_name) - puts(traceback.format_exc(e)) + puts(traceback.format_exc()) sys.exit(1) - except Exception, e: + except Exception: error("Unable to bootstrap storage!") - puts(traceback.format_exc(e)) + puts(traceback.format_exc()) sys.exit(1) + @yappi_profile(return_callback=yappi_aggregate) + def bootstrap_pubsub_mixin(self): + puts("Bootstrapping pubsub...") + try: + self.bootstrap_pubsub() + # Make sure settings are there. + self.pubsub.verify_settings() + with indent(2): + show_valid("Bootstrapped!") + puts("") + except ImportError as e: + module_name = traceback.format_exc().split(" ")[-1] + error("Unable to bootstrap pubsub - attempting to load %s" % module_name) + puts(traceback.format_exc()) + sys.exit(1) + except Exception as e: + error("Unable to bootstrap pubsub!") + puts(traceback.format_exc()) + sys.exit(1) + + @yappi_profile(return_callback=yappi_aggregate) def bootstrap_scheduler(self): bootstrapped = False try: @@ -343,17 +756,18 @@ def bootstrap_scheduler(self): meta["num_times_per_day"] ) bootstrapped = True - except Exception, e: + except Exception as e: self.startup_error("Error bootstrapping scheduler", e) if bootstrapped: show_valid("Scheduler started.") self.scheduler.start_loop(self) + @yappi_profile(return_callback=yappi_aggregate) def bootstrap_bottle(self): bootstrapped = False try: for cls, function_name in self.bottle_routes: - instantiated_cls = cls() + instantiated_cls = cls(bot=self) instantiated_fn = getattr(instantiated_cls, function_name) bottle_route_args = {} for k, v in instantiated_fn.will_fn_metadata.items(): @@ -361,31 +775,115 @@ def bootstrap_bottle(self): bottle_route_args[k[len("bottle_"):]] = v bottle.route(instantiated_fn.will_fn_metadata["bottle_route"], **bottle_route_args)(instantiated_fn) bootstrapped = True - except Exception, e: + except Exception as e: self.startup_error("Error bootstrapping bottle", e) if bootstrapped: - show_valid("Web server started.") + show_valid("Web server started at %s." % (settings.PUBLIC_URL,)) bottle.run(host='0.0.0.0', port=settings.HTTPSERVER_PORT, server='cherrypy', quiet=True) - def bootstrap_xmpp(self): - bootstrapped = False - try: - self.start_xmpp_client() - sorted_help = {} - for k, v in self.help_modules.items(): - sorted_help[k] = sorted(v) + @yappi_profile(return_callback=yappi_aggregate) + def bootstrap_io(self): + # puts("Bootstrapping IO...") + self.has_stdin_io_backend = False + self.io_backends = [] + self.io_threads = [] + self.stdin_io_backends = [] + for b in self.valid_io_backends: + module = import_module(b) + for class_name, cls in inspect.getmembers(module, predicate=inspect.isclass): + try: + if ( + hasattr(cls, "is_will_iobackend") and + cls.is_will_iobackend and + class_name != "IOBackend" and + class_name != "StdInOutIOBackend" + ): + c = cls() - self.save("help_modules", sorted_help) - self.save("all_listener_regexes", self.all_listener_regexes) - self.connect() - bootstrapped = True - except Exception, e: - self.startup_error("Error bootstrapping xmpp", e) - if bootstrapped: - show_valid("Chat client started.") - show_valid("Will is running.") - self.process(block=True) + if hasattr(c, "stdin_process") and c.stdin_process: + thread = Process( + target=c._start, + args=(b,), + ) + thread.start() + self.has_stdin_io_backend = True + self.io_threads.append(thread) + else: + thread = Process( + target=c._start, + args=( + b, + ) + ) + thread.start() + self.io_threads.append(thread) + + show_valid("IO: %s Backend started." % cls.friendly_name) + except Exception as e: + self.startup_error("Error bootstrapping %s io" % b, e) + + self.io_backends.append(b) + + @yappi_profile(return_callback=yappi_aggregate) + def bootstrap_analysis(self): + + self.analysis_backends = [] + self.analysis_threads = [] + + for b in settings.ANALYZE_BACKENDS: + module = import_module(b) + for class_name, cls in inspect.getmembers(module, predicate=inspect.isclass): + try: + if ( + hasattr(cls, "is_will_analysisbackend") and + cls.is_will_analysisbackend and + class_name != "AnalysisBackend" + ): + c = cls() + thread = Process( + target=c.start, + args=(b,), + kwargs={"bot": self}, + ) + thread.start() + self.analysis_threads.append(thread) + show_valid("Analysis: %s Backend started." % cls.__name__) + except Exception as e: + self.startup_error("Error bootstrapping %s io" % b, e) + + self.analysis_backends.append(b) + pass + + @yappi_profile(return_callback=yappi_aggregate) + def bootstrap_generation(self): + self.generation_backends = [] + self.generation_threads = [] + for b in settings.GENERATION_BACKENDS: + module = import_module(b) + for class_name, cls in inspect.getmembers(module, predicate=inspect.isclass): + try: + if ( + hasattr(cls, "is_will_generationbackend") and + cls.is_will_generationbackend and + class_name != "GenerationBackend" + ): + c = cls() + thread = Process( + target=c.start, + args=(b,), + kwargs={"bot": self}, + ) + thread.start() + self.generation_threads.append(thread) + show_valid("Generation: %s Backend started." % cls.__name__) + except Exception as e: + self.startup_error("Error bootstrapping %s io" % b, e) + + self.generation_backends.append(b) + pass + + @yappi_profile(return_callback=yappi_aggregate) def bootstrap_plugins(self): puts("Bootstrapping plugins...") OTHER_HELP_HEADING = "Other" @@ -401,39 +899,33 @@ def bootstrap_plugins(self): if f[-3:] == ".py" and f != "__init__.py": try: module_path = os.path.join(root, f) - path_components = os.path.split(module_path) + path_components = module_path.split(os.sep) module_name = path_components[-1][:-3] full_module_name = ".".join(path_components) - # Need to pass along module name, path all the way through - combined_name = ".".join([plugin_name, module_name]) # Check blacklist. blacklisted = False for b in settings.PLUGIN_BLACKLIST: - if b in combined_name: + if b in full_module_name: blacklisted = True - - try: - plugin_modules[full_module_name] = imp.load_source(module_name, module_path) - except: - # If it's blacklisted, don't worry if this blows up. - if blacklisted: - pass - else: - raise + break parent_mod = path_components[-2].split("/")[-1] parent_help_text = parent_mod.title() - try: - parent_root = os.path.join(root, "__init__.py") - parent = imp.load_source(parent_mod, parent_root) - parent_help_text = getattr(parent, "MODULE_DESCRIPTION", parent_help_text) - except: - # If it's blacklisted, don't worry if this blows up. - if blacklisted: - pass - else: - raise + # Don't even *try* to load a blacklisted module. + if not blacklisted: + try: + plugin_modules[full_module_name] = imp.load_source(module_name, module_path) + + parent_root = os.path.join(root, "__init__.py") + parent = imp.load_source(parent_mod, parent_root) + parent_help_text = getattr(parent, "MODULE_DESCRIPTION", parent_help_text) + except: + # If it's blacklisted, don't worry if this blows up. + if blacklisted: + pass + else: + raise plugin_modules_library[full_module_name] = { "full_module_name": full_module_name, @@ -444,7 +936,7 @@ def bootstrap_plugins(self): "parent_help_text": parent_help_text, "blacklisted": blacklisted, } - except Exception, e: + except Exception as e: self.startup_error("Error loading %s" % (module_path,), e) self.plugins = [] @@ -459,19 +951,20 @@ def bootstrap_plugins(self): "module": module, "full_module_name": name, "parent_name": plugin_modules_library[name]["parent_name"], + "parent_path": plugin_modules_library[name]["file_path"], "parent_module_name": plugin_modules_library[name]["parent_module_name"], "parent_help_text": plugin_modules_library[name]["parent_help_text"], "blacklisted": plugin_modules_library[name]["blacklisted"], }) - except Exception, e: + except Exception as e: self.startup_error("Error bootstrapping %s" % (class_name,), e) - except Exception, e: + except Exception as e: self.startup_error("Error bootstrapping %s" % (name,), e) self._plugin_modules_library = plugin_modules_library # Sift and Sort. - self.message_listeners = [] + self.message_listeners = {} self.periodic_tasks = [] self.random_tasks = [] self.bottle_routes = [] @@ -496,6 +989,7 @@ def bootstrap_plugins(self): last_parent_name = plugin_info["parent_help_text"] with indent(2): plugin_name = plugin_info["name"] + plugin_warnings = [] # Just a little nicety if plugin_name[-6:] == "Plugin": plugin_name = plugin_name[:-6] @@ -505,13 +999,15 @@ def bootstrap_plugins(self): plugin_instances = {} for function_name, fn in inspect.getmembers( plugin_info["class"], - predicate=inspect.ismethod + predicate=lambda x: inspect.ismethod(x) or inspect.isfunction(x) ): try: # Check for required_settings with indent(2): if hasattr(fn, "will_fn_metadata"): meta = fn.will_fn_metadata + if "warnings" in meta: + plugin_warnings.append(meta["warnings"]) if "required_settings" in meta: for s in meta["required_settings"]: self.required_settings_from_plugins[s] = { @@ -530,17 +1026,17 @@ def bootstrap_plugins(self): regex = "(?i)%s" % regex help_regex = meta["listener_regex"] if meta["listens_only_to_direct_mentions"]: - help_regex = "@%s %s" % (settings.HANDLE, help_regex) + help_regex = "@%s %s" % (settings.WILL_HANDLE, help_regex) self.all_listener_regexes.append(help_regex) if meta["__doc__"]: pht = plugin_info.get("parent_help_text", None) if pht: if pht in self.help_modules: - self.help_modules[pht].append(meta["__doc__"]) + self.help_modules[pht].append(u"%s" % meta["__doc__"]) else: - self.help_modules[pht] = [meta["__doc__"]] + self.help_modules[pht] = [u"%s" % meta["__doc__"]] else: - self.help_modules[OTHER_HELP_HEADING].append(meta["__doc__"]) + self.help_modules[OTHER_HELP_HEADING].append(u"%s" % meta["__doc__"]) if meta["multiline"]: compiled_regex = re.compile(regex, re.MULTILINE | re.DOTALL) else: @@ -549,10 +1045,15 @@ def bootstrap_plugins(self): if plugin_info["class"] in plugin_instances: instance = plugin_instances[plugin_info["class"]] else: - instance = plugin_info["class"]() + instance = plugin_info["class"](bot=self) plugin_instances[plugin_info["class"]] = instance - self.message_listeners.append({ + full_method_name = "%s.%s" % (plugin_info["name"], function_name) + cleaned_info = copy.copy(plugin_info) + del cleaned_info["module"] + del cleaned_info["class"] + self.message_listeners[full_method_name] = { + "full_method_name": full_method_name, "function_name": function_name, "class_name": plugin_info["name"], "regex_pattern": meta["listener_regex"], @@ -560,10 +1061,13 @@ def bootstrap_plugins(self): "fn": getattr(instance, function_name), "args": meta["listener_args"], "include_me": meta["listener_includes_me"], + "case_sensitive": meta["case_sensitive"], + "multiline": meta["multiline"], "direct_mentions_only": meta["listens_only_to_direct_mentions"], "admin_only": meta["listens_only_to_admin"], "acl": meta["listeners_acl"], - }) + "plugin_info": cleaned_info, + } if meta["listener_includes_me"]: self.some_listeners_include_me = True elif "periodic_task" in meta and meta["periodic_task"]: @@ -575,7 +1079,8 @@ def bootstrap_plugins(self): elif "bottle_route" in meta: # puts("- %s" % function_name) self.bottle_routes.append((plugin_info["class"], function_name)) - except Exception, e: + + except Exception as e : error(plugin_name) self.startup_error( "Error bootstrapping %s.%s" % ( @@ -583,7 +1088,13 @@ def bootstrap_plugins(self): function_name, ), e ) - show_valid(plugin_name) - except Exception, e: + if len(plugin_warnings) > 0: + show_invalid(plugin_name) + for w in plugin_warnings: + warn(w) + else: + show_valid(plugin_name) + except Exception as e: self.startup_error("Error bootstrapping %s" % (plugin_info["class"],), e) + self.save("all_listener_regexes", self.all_listener_regexes) puts("") diff --git a/will/mixins/__init__.py b/will/mixins/__init__.py index b27e2434..9bf30183 100644 --- a/will/mixins/__init__.py +++ b/will/mixins/__init__.py @@ -1,10 +1,15 @@ -from errors import ErrorMixin -from email import EmailMixin -from hipchat import HipChatMixin -from naturaltime import NaturalTimeMixin -from room import RoomMixin -from roster import RosterMixin -from plugins_library import PluginModulesLibraryMixin -from schedule import ScheduleMixin -from settings import SettingsMixin -from storage import StorageMixin +from will.mixins.errors import ErrorMixin +from will.mixins.encryption import EncryptionMixin +from will.mixins.email import EmailMixin +from will.mixins.naturaltime import NaturalTimeMixin +from will.mixins.plugins_library import PluginModulesLibraryMixin +from will.mixins.schedule import ScheduleMixin +from will.mixins.settings import SettingsMixin +from will.mixins.sleep import SleepMixin +from will.mixins.storage import StorageMixin +from will.mixins.pubsub import PubSubMixin + +# RosterMixin and RoomMixin have been moved to from will.backends.io_adapters.hipchat +# This is just for logging a warning for people who have used it internally +from will.mixins.room import RoomMixin +from will.mixins.roster import RosterMixin diff --git a/will/mixins/email.py b/will/mixins/email.py index 5891bc6a..efe656ef 100644 --- a/will/mixins/email.py +++ b/will/mixins/email.py @@ -1,7 +1,6 @@ import requests from will import settings -from will.decorators import require_settings class EmailMixin(object): @@ -16,7 +15,7 @@ def send_email(self, from_email=None, email_list=[], subject="", message=""): else: raise ValueError("Couldn't send email, from_email was None and there was no DEFAULT_FROM_EMAIL") - if email_list is None or len(email_list) == 0: + if not email_list: raise ValueError("Email list wasn't specified. Expecting a list of emails, got %s" % email_list) api_url = getattr(settings, 'MAILGUN_API_URL', None) diff --git a/will/mixins/encryption.py b/will/mixins/encryption.py new file mode 100644 index 00000000..8a592658 --- /dev/null +++ b/will/mixins/encryption.py @@ -0,0 +1,31 @@ +import importlib +import logging +import dill as pickle +import functools +from will import settings + + +class EncryptionMixin(object): + @property + def encryption_backend(self): + if not hasattr(self, "_encryption"): + if hasattr(self, "bot") and hasattr(self.bot, "_encryption"): + self._encryption = self.bot._encryption + else: + # The ENCRYPTION_BACKEND setting points to a specific module namespace + # aes => will.encryption.aes + module_name = ''.join([ + 'will.backends.encryption.', + getattr(settings, 'ENCRYPTION_BACKEND', 'aes'), + ]) + encryption_module = importlib.import_module(module_name) + self._encryption = encryption_module.bootstrap(settings) + return self._encryption + + def encrypt(self, raw): + return self.encryption_backend.encrypt_to_b64(raw) + + def decrypt(self, enc): + if enc: + return self.encryption_backend.decrypt_from_b64(enc) + return None diff --git a/will/mixins/errors.py b/will/mixins/errors.py index c5179dc2..5a28f2c9 100644 --- a/will/mixins/errors.py +++ b/will/mixins/errors.py @@ -14,6 +14,7 @@ def add_startup_error(self, error_message): self._startup_errors.append(error_message) def startup_error(self, error_message, exception_instance): + traceback.print_exc() error_message = "%s%s" % ( error_message, ":\n\n%s\nContinuing...\n" % traceback.format_exc(exception_instance) diff --git a/will/mixins/hipchat.py b/will/mixins/hipchat.py index 1e0c3da5..9aeb7c7c 100644 --- a/will/mixins/hipchat.py +++ b/will/mixins/hipchat.py @@ -1,118 +1,10 @@ -import json -import logging -import requests -import traceback - -from will import settings - -ROOM_NOTIFICATION_URL = "https://%(server)s/v2/room/%(room_id)s/notification?auth_token=%(token)s" -ROOM_TOPIC_URL = "https://%(server)s/v2/room/%(room_id)s/topic?auth_token=%(token)s" -PRIVATE_MESSAGE_URL = "https://%(server)s/v2/user/%(user_id)s/message?auth_token=%(token)s" -SET_TOPIC_URL = "https://%(server)s/v2/room/%(room_id)s/topic?auth_token=%(token)s" -USER_DETAILS_URL = "https://%(server)s/v2/user/%(user_id)s?auth_token=%(token)s" -ALL_USERS_URL = "https://%(server)s/v2/user?auth_token=%(token)s&start-index=%(start_index)s&max-results=%(max_results)s" - - class HipChatMixin(object): - def send_direct_message(self, user_id, message_body, html=False, notify=False, **kwargs): - if kwargs: - logging.warn("Unknown keyword args for send_direct_message: %s" % kwargs) - - format = "text" - if html: - format = "html" - - try: - # https://www.hipchat.com/docs/apiv2/method/private_message_user - url = PRIVATE_MESSAGE_URL % {"server": settings.HIPCHAT_SERVER, - "user_id": user_id, - "token": settings.V2_TOKEN} - data = { - "message": message_body, - "message_format": format, - "notify": notify, - } - headers = {'Content-type': 'application/json', 'Accept': 'text/plain'} - requests.post(url, headers=headers, data=json.dumps(data), **settings.REQUESTS_OPTIONS) - except: - logging.critical("Error in send_direct_message: \n%s" % traceback.format_exc()) - - def send_direct_message_reply(self, message, message_body): - try: - message.reply(message_body).send() - except: - logging.critical("Error in send_direct_message_reply: \n%s" % traceback.format_exc()) - - def send_room_message(self, room_id, message_body, html=False, color="green", notify=False, **kwargs): - if kwargs: - logging.warn("Unknown keyword args for send_room_message: %s" % kwargs) - - format = "text" - if html: - format = "html" - - try: - # https://www.hipchat.com/docs/apiv2/method/send_room_notification - url = ROOM_NOTIFICATION_URL % {"server": settings.HIPCHAT_SERVER, - "room_id": room_id, - "token": settings.V2_TOKEN} - data = { - "message": message_body, - "message_format": format, - "color": color, - "notify": notify, - } - headers = {'Content-type': 'application/json', 'Accept': 'text/plain'} - requests.post(url, headers=headers, data=json.dumps(data), **settings.REQUESTS_OPTIONS) - except: - logging.critical("Error in send_room_message: \n%s" % traceback.format_exc()) - - def set_room_topic(self, room_id, topic): - try: - # https://www.hipchat.com/docs/apiv2/method/send_room_notification - url = ROOM_TOPIC_URL % {"server": settings.HIPCHAT_SERVER, - "room_id": room_id, - "token": settings.V2_TOKEN} - data = { - "topic": topic, - } - headers = {'Content-type': 'application/json', 'Accept': 'text/plain'} - requests.put(url, headers=headers, data=json.dumps(data), **settings.REQUESTS_OPTIONS) - except: - logging.critical("Error in set_room_topic: \n%s" % traceback.format_exc()) - - def get_hipchat_user(self, user_id, q=None): - url = USER_DETAILS_URL % {"server": settings.HIPCHAT_SERVER, - "user_id": user_id, - "token": settings.V2_TOKEN} - r = requests.get(url, **settings.REQUESTS_OPTIONS) - if q: - q.put(r.json()) - else: - return r.json() - - @property - def full_hipchat_user_list(self): - if not hasattr(self, "_full_hipchat_user_list"): - full_roster = {} - - # Grab the first roster page, and populate full_roster - url = ALL_USERS_URL % {"server": settings.HIPCHAT_SERVER, - "token": settings.V2_TOKEN, - "start_index": 0, - "max_results": 1000} - r = requests.get(url, **settings.REQUESTS_OPTIONS) - for user in r.json()['items']: - full_roster["%s" % (user['id'],)] = user - - # Keep going through the next pages until we're out of pages. - while 'next' in r.json()['links']: - url = "%s&auth_token=%s" % (r.json()['links']['next'], settings.V2_TOKEN) - r = requests.get(url, **settings.REQUESTS_OPTIONS) - - for user in r.json()['items']: - full_roster["%s" % (user['id'],)] = user - - self._full_hipchat_user_list = full_roster - return self._full_hipchat_user_list + def __init__(self, *args, **kwargs): + import logging + logging.critical( + "HipChatMixin functionality has been moved to the hipchat backend.\n" + + "If you need functionality that used to be on this class, please either\n" + + "publish messages, or use will.backends.io_backends.hipchat:HipChatBackend." + ) + super(HipChatMixin, self).__init__(*args, **kwargs) diff --git a/will/mixins/plugins_library.py b/will/mixins/plugins_library.py index 005352e8..d44f1560 100644 --- a/will/mixins/plugins_library.py +++ b/will/mixins/plugins_library.py @@ -13,6 +13,6 @@ def plugin_modules_library(self): else: self._plugin_modules_library = self.load("plugin_modules_library", {}) except: - logging.critical("Error loading plugin_modules_library: \n%s" % traceback.format_exc()) + logging.critical("Error loading plugin_modules_library: \n%s", traceback.format_exc()) return {} return self._plugin_modules_library diff --git a/will/mixins/pubsub.py b/will/mixins/pubsub.py new file mode 100644 index 00000000..d40a1abd --- /dev/null +++ b/will/mixins/pubsub.py @@ -0,0 +1,48 @@ +import importlib +import logging +import dill as pickle +import functools +from will import settings + + +class PubSubMixin(object): + def bootstrap_pubsub(self): + if not hasattr(self, "pubsub"): + if hasattr(self, "bot") and hasattr(self.bot, "pubsub"): + self.pubsub = self.bot.pubsub + else: + # The PUBSUB_BACKEND setting points to a specific module namespace + # redis => will.pubsub.redis_backend + # zeromq => will.pubsub.zeromq_backend + # etc... + module_name = ''.join([ + 'will.backends.pubsub.', + getattr(settings, 'PUBSUB_BACKEND', 'redis'), + '_pubsub' + ]) + pubsub_module = importlib.import_module(module_name) + + # Now create our pubsub object using the bootstrap function + # from within the import + self.pubsub = pubsub_module.bootstrap(settings) + + def subscribe(self, topic): + self.bootstrap_pubsub() + try: + return self.pubsub.subscribe(topic) + except Exception: + logging.exception("Unable to subscribe to %s", topic) + + def publish(self, topic, obj): + self.bootstrap_pubsub() + try: + return self.pubsub.publish(topic, obj) + except Exception: + logging.exception("Unable to publish %s to %s", (obj, topic)) + + def unsubscribe(self, topic): + self.bootstrap_pubsub() + try: + return self.pubsub.unsubscribe(topic) + except Exception: + logging.exception("Unable to unsubscribe to %s", topic) diff --git a/will/mixins/room.py b/will/mixins/room.py index 1048067d..810a073b 100644 --- a/will/mixins/room.py +++ b/will/mixins/room.py @@ -1,116 +1,21 @@ -from datetime import datetime -import logging -import json -import requests +class Room(object): -from will import settings -from will.utils import Bunch - -logger = logging.getLogger(__name__) - -V1_TOKEN_URL = "https://%(server)s/v1/rooms/list?auth_token=%(token)s" -V2_TOKEN_URL = "https://%(server)s/v2/room?auth_token=%(token)s&expand=items" - - -class Room(Bunch): - - @property - def history(self): - payload = {"auth_token": settings.V2_TOKEN} - room_id = int(self['id']) - response = requests.get("https://{1}/v2/room/{0}/history".format(str(room_id), - settings.HIPCHAT_SERVER), - params=payload, **settings.REQUESTS_OPTIONS) - data = json.loads(response.text)['items'] - for item in data: - item['date'] = datetime.strptime(item['date'][:-13], "%Y-%m-%dT%H:%M:%S") - return data - - @property - def participants(self): - payload = {"auth_token": settings.V2_TOKEN} - room_id = int(self['id']) - response = requests.get( - "https://{1}/v2/room/{0}/participant".format( - str(room_id), - settings.HIPCHAT_SERVER - ), - params=payload, - **settings.REQUESTS_OPTIONS - ).json() - data = response['items'] - while 'next' in response['links']: - response = requests.get(response['links']['next'], - params=payload, **settings.REQUESTS_OPTIONS).json() - data.extend(response['items']) - return data + def __init__(self, *args, **kwargs): + import logging + logging.critical( + "Room has been renamed to HipChatRoom, and will be removed from future releases.\n" + + "Please change all your imports to will.backends.io_adapters.hipchat import HipChatRoom" + ) + super(Room, self).__init__(*args, **kwargs) class RoomMixin(object): - def update_available_rooms(self, q=None): - self._available_rooms = {} - # Use v1 token to grab a full room list if we can (good to avoid rate limiting) - if hasattr(settings, "V1_TOKEN"): - url = V1_TOKEN_URL % {"server": settings.HIPCHAT_SERVER, - "token": settings.V1_TOKEN} - r = requests.get(url, **settings.REQUESTS_OPTIONS) - if r.status_code == requests.codes.unauthorized: - raise Exception("V1_TOKEN authentication failed with HipChat") - for room in r.json()["rooms"]: - self._available_rooms[room["name"]] = room - # Otherwise, grab 'em one-by-one via the v2 api. - else: - params = {} - params['start-index'] = 0 - max_results = params['max-results'] = 1000 - url = V2_TOKEN_URL % {"server": settings.HIPCHAT_SERVER, - "token": settings.V2_TOKEN} - while True: - resp = requests.get(url, params=params, - **settings.REQUESTS_OPTIONS) - if resp.status_code == requests.codes.unauthorized: - raise Exception("V2_TOKEN authentication failed with HipChat") - rooms = resp.json() - - for room in rooms["items"]: - room["room_id"] = room["id"] - self._available_rooms[room["name"]] = Room(**room) - - logger.info('Got %d rooms', len(rooms['items'])) - if len(rooms['items']) == max_results: - params['start-index'] += max_results - else: - break - - self.save("hipchat_rooms", self._available_rooms) - if q: - q.put(self._available_rooms) - - @property - def available_rooms(self): - if not hasattr(self, "_available_rooms"): - self._available_rooms = self.load('hipchat_rooms', None) - if not self._available_rooms: - self.update_available_rooms() - - return self._available_rooms - - def get_room_by_jid(self, jid): - for name, room in self.available_rooms.items(): - if "xmpp_jid" in room and room["xmpp_jid"] == jid: - return room - return None - - def get_room_from_message(self, message): - return self.get_room_by_jid(message.getMucroom()) - def get_room_from_name_or_id(self, name_or_id): - for name, room in self.available_rooms.items(): - if name_or_id == name: - return room - if "xmpp_jid" in room and name_or_id == room["xmpp_jid"]: - return room - if "room_id" in room and name_or_id == room["room_id"]: - return room - return None + def __init__(self, *args, **kwargs): + import logging + logging.critical( + "RoomMixin has been renamed to HipChatRoomMixin, and will be removed from future releases.\n" + + "Please change all your imports to will.backends.io_adapters.hipchat import HipChatRoomMixin" + ) + super(RoomMixin, self).__init__(*args, **kwargs) diff --git a/will/mixins/roster.py b/will/mixins/roster.py index 64609580..d6b207c3 100644 --- a/will/mixins/roster.py +++ b/will/mixins/roster.py @@ -1,51 +1,9 @@ -from ..utils import is_admin -from ..acl import is_acl_allowed - - class RosterMixin(object): - @property - def internal_roster(self): - if not hasattr(self, "_internal_roster"): - self._internal_roster = self.load('will_roster', {}) - return self._internal_roster - - def get_user_by_full_name(self, name): - for jid, info in self.internal_roster.items(): - if info["name"] == name: - return info - return None - - def get_user_by_nick(self, nick): - for jid, info in self.internal_roster.items(): - if info["nick"] == nick: - return info - return None - - def get_user_by_jid(self, jid): - if jid in self.internal_roster: - return self.internal_roster[jid] - - return None - - def get_user_from_message(self, message): - if message["type"] == "groupchat": - return self.get_user_by_full_name(message["mucnick"]) - elif message['type'] in ('chat', 'normal'): - jid = ("%s" % message["from"]).split("/")[0] - return self.get_user_by_jid(jid) - else: - return None - - def message_is_from_admin(self, message): - nick = self.get_user_from_message(message)['nick'] - return is_admin(nick) - - def message_is_allowed(self, message, acl): - nick = self.get_user_from_message(message)['nick'] - return is_acl_allowed(nick, acl) - def get_user_by_hipchat_id(self, id): - for jid, info in self.internal_roster.items(): - if info["hipchat_id"] == id: - return info - return None + def __init__(self, *args, **kwargs): + import logging + logging.critical( + "RosterMixin has been moved to the hipchat backend.\n" + + "Please change all your imports to `from will.backends.io_adapters.hipchat import HipChatRosterMixin`" + ) + super(RosterMixin, self).__init__(*args, **kwargs) diff --git a/will/mixins/schedule.py b/will/mixins/schedule.py index d5695f96..736a2ca5 100644 --- a/will/mixins/schedule.py +++ b/will/mixins/schedule.py @@ -4,9 +4,10 @@ import time import traceback from apscheduler.triggers.cron import CronTrigger +from will.mixins.pubsub import PubSubMixin -class ScheduleMixin(object): +class ScheduleMixin(PubSubMixin, object): def times_key(self, periodic_list=False): if periodic_list: @@ -19,21 +20,20 @@ def schedule_key(self, periodic_list=False): return "will_schedule_list" def get_schedule_list(self, periodic_list=False): - # TODO: Clean this up. return self.load(self.schedule_key(periodic_list=periodic_list), {}) def save_schedule_list(self, new_list, periodic_list=False): self.save(self.schedule_key(periodic_list=periodic_list), new_list) def get_times_list(self, periodic_list=False): - # TODO: Clean this up. return self.load(self.times_key(periodic_list=periodic_list), {}) def save_times_list(self, new_list, periodic_list=False): return self.save(self.times_key(periodic_list=periodic_list), new_list) - def add_direct_message_to_schedule(self, when, content, message, *args, **kwargs): - target_user = self.get_user_from_message(message) + # TODO: Create new version of this that's properly abstracted, instead of get_user_from_message + def add_direct_message_to_schedule(self, when, content, message, target_user, *args, **kwargs): + # target_user = self.get_user_from_message(message) self.add_to_schedule(when, { "type": "direct_message", "content": content, @@ -51,6 +51,9 @@ def add_room_message_to_schedule(self, when, content, room, *args, **kwargs): "kwargs": kwargs, }) + def add_outgoing_event_to_schedule(self, when, event, *args, **kwargs): + self.add_to_schedule(when, event, *args, **kwargs) + def add_to_schedule(self, when, item, periodic_list=False, ignore_scheduler_lock=False): try: while ( @@ -74,8 +77,9 @@ def add_to_schedule(self, when, item, periodic_list=False, ignore_scheduler_lock except: logging.critical( - "Error adding to schedule at %s. \n\n%s\nContinuing...\n" % - (when, traceback.format_exc()) + "Error adding to schedule at %s. \n\n%s\nContinuing...\n", + when, + traceback.format_exc() ) self.save("scheduler_add_lock", False) @@ -93,6 +97,8 @@ def add_periodic_task(self, module_name, cls_name, function_name, sched_args, now = datetime.datetime.now() ct = CronTrigger(*sched_args, **sched_kwargs) when = ct.get_next_fire_time(now) + logging.info("ct.get_next_fire_time(now)") + logging.info(when) item = { "type": "periodic_task", "module_name": module_name, diff --git a/will/mixins/settings.py b/will/mixins/settings.py index 579482b0..94a3cfd9 100644 --- a/will/mixins/settings.py +++ b/will/mixins/settings.py @@ -1,5 +1,10 @@ +from clint.textui import colored, puts, indent +from will import settings +from will.utils import show_valid, warn, error + class SettingsMixin(object): + required_settings = [] def verify_setting_exists(self, setting_name, message=None): from will import settings @@ -8,3 +13,33 @@ def verify_setting_exists(self, setting_name, message=None): self.say("%s not set." % setting_name, message=message) return False return True + + def verify_settings(self, quiet=False): + passed = True + for s in self.required_settings: + if not hasattr(settings, s["name"]): + meta = s + if hasattr(self, "friendly_name"): + meta["friendly_name"] = self.friendly_name + else: + meta["friendly_name"] = self.__class__.__name__ + if not quiet: + with indent(2): + error("%(name)s is missing. It's required by the %(friendly_name)s backend." % meta) + with indent(2): + error_message = ( + "To obtain a %(name)s: \n%(obtain_at)s" + ) % meta + puts(error_message) + passed = False + # raise Exception(error_message) + else: + if not quiet: + with indent(2): + show_valid(s["name"]) + if not passed: + raise EnvironmentError( + "Missing required settings when starting up %s." + "Please fix the error above and restart Will!" % (meta["friendly_name"], ) + ) + return passed diff --git a/will/mixins/sleep.py b/will/mixins/sleep.py new file mode 100644 index 00000000..2dc244d2 --- /dev/null +++ b/will/mixins/sleep.py @@ -0,0 +1,15 @@ +import random +import time +import sys +from will import settings + + +class SleepMixin(object): + def sleep_for_event_loop(self, multiplier=1): + try: + if not hasattr(self, "sleep_time"): + self.sleep_time = settings.EVENT_LOOP_INTERVAL + (random.randint(0, 1) * settings.EVENT_LOOP_INTERVAL) + + time.sleep(self.sleep_time * multiplier) + except KeyboardInterrupt: + sys.exit(0) diff --git a/will/mixins/storage.py b/will/mixins/storage.py index acb449c5..6cfd4804 100644 --- a/will/mixins/storage.py +++ b/will/mixins/storage.py @@ -1,8 +1,8 @@ import importlib import logging import dill as pickle -import functools from will import settings +from will.abstractions import Person, Event, Channel, Message class StorageMixin(object): @@ -16,9 +16,9 @@ def bootstrap_storage(self): # couchbase => will.storage.couchbase_backend # etc... module_name = ''.join([ - 'will.storage.', + 'will.backends.storage.', getattr(settings, 'STORAGE_BACKEND', 'redis'), - '_storage' + '_backend' ]) storage_module = importlib.import_module(module_name) @@ -30,21 +30,21 @@ def save(self, key, value, expire=None): self.bootstrap_storage() try: return self.storage.save(key, pickle.dumps(value), expire=expire) - except Exception: + except: logging.exception("Unable to save %s", key) def clear(self, key): self.bootstrap_storage() try: return self.storage.clear(key) - except Exception: + except: logging.exception("Unable to clear %s", key) def clear_all_keys(self): self.bootstrap_storage() try: return self.storage.clear_all_keys() - except Exception: + except: logging.exception("Unable to clear all keys") def load(self, key, default=None): @@ -54,8 +54,9 @@ def load(self, key, default=None): if val is not None: return pickle.loads(val) return default - except Exception: - logging.exception("Failed to load %s", key) + except: + # logging.exception("Failed to load %s", key) + return default def size(self): self.bootstrap_storage() @@ -63,3 +64,21 @@ def size(self): return self.storage.size() except Exception: logging.exception("Failed to get the size of our storage") + + # list specific save/load/clear operations + + def pop(self, key, value): + tmp_value = self.load(key) + if tmp_value is None: + pass + else: + tmp_value.remove(value) + self.save(key, tmp_value) + + def append(self, key, value, expire=None): + tmp_value = self.load(key) + if tmp_value is None: + self.save(key, [value], expire) + else: + tmp_value.append(value) + self.save(key, tmp_value, expire) diff --git a/will/plugin.py b/will/plugin.py index a3ca5087..9b995541 100644 --- a/will/plugin.py +++ b/will/plugin.py @@ -1,103 +1,202 @@ import re import logging -import settings from bottle import request -from mixins import NaturalTimeMixin, RosterMixin, RoomMixin, ScheduleMixin, HipChatMixin, StorageMixin, SettingsMixin, \ - EmailMixin -from utils import html_to_text +from will import settings +from will.abstractions import Event, Message +# Backwards compatability with 1.x, eventually to be deprecated. +from will.backends.io_adapters.hipchat import HipChatRosterMixin, HipChatRoomMixin +from will.mixins import NaturalTimeMixin, ScheduleMixin, StorageMixin, SettingsMixin, \ + EmailMixin, PubSubMixin +from will.utils import html_to_text -class WillPlugin(EmailMixin, StorageMixin, NaturalTimeMixin, RoomMixin, RosterMixin, - ScheduleMixin, HipChatMixin, SettingsMixin): + +class WillPlugin(EmailMixin, StorageMixin, NaturalTimeMixin, HipChatRoomMixin, HipChatRosterMixin, + ScheduleMixin, SettingsMixin, PubSubMixin): is_will_plugin = True request = request - def _rooms_from_message_and_room(self, message, room): - if room == "ALL_ROOMS": - rooms = self.available_rooms - elif room: - rooms = [self.get_room_from_name_or_id(room), ] - else: - if message: - rooms = [self.get_room_from_message(message), ] - else: - rooms = [self.get_room_from_name_or_id(settings.DEFAULT_ROOM), ] - return rooms + def __init__(self, *args, **kwargs): + if "bot" in kwargs: + self.bot = kwargs["bot"] + del kwargs["bot"] + if "message" in kwargs: + self.message = kwargs["message"] + del kwargs["message"] + + super(WillPlugin, self).__init__(*args, **kwargs) def _prepared_content(self, content, message, kwargs): - if kwargs is None: - kwargs = {} - - if kwargs.get("html", False) and (message and message['type'] in ('chat', 'normal')): - # 1-1 can't have HTML. - content = html_to_text(content) - elif kwargs.get("html", True): - # Hipchat is weird about spaces between tags. - content = re.sub(r'>\s+<', '><', content) + content = re.sub(r'>\s+<', '><', content) return content - def say(self, content, message=None, room=None, **kwargs): - # Valid kwargs: - # color: yellow, red, green, purple, gray, random. Default is green. - # html: Display HTML or not. Default is False - # notify: Ping everyone. Default is False - - content = self._prepared_content(content, message, kwargs) - rooms = [] - if room is not None: - try: - room_id = room["room_id"] - except KeyError: - logging.error(u'"{0}" is not a room object.'.format(room)) + def _trim_for_execution(self, message): + # Trim it down + if hasattr(message, "analysis"): + message.analysis = None + if hasattr(message, "source_message") and hasattr(message.source_message, "analysis"): + message.source_message.analysis = None + return message + + def get_backend(self, message, service=None): + backend = False + if service: + for b in settings.IO_BACKENDS: + if service in b: + return b + + if hasattr(message, "backend"): + backend = message.backend + elif message and hasattr(message, "data") and hasattr(message.data, "backend"): + backend = message.data.backend + else: + backend = settings.DEFAULT_BACKEND + return backend + + def get_message(self, message_passed): + if not message_passed and hasattr(self, "message"): + return self.message + return message_passed + + def say(self, content, message=None, room=None, channel=None, service=None, package_for_scheduling=False, **kwargs): + logging.info("self.say") + logging.info(content) + if channel: + room = channel + elif room: + channel = room + + if not "channel" in kwargs and channel: + kwargs["channel"] = channel + + message = self.get_message(message) + message = self._trim_for_execution(message) + backend = self.get_backend(message, service=service) + + if backend: + e = Event( + type="say", + content=content, + source_message=message, + kwargs=kwargs, + ) + if package_for_scheduling: + return "message.outgoing.%s" % backend, e else: - self.send_room_message(room_id, content, **kwargs) - elif message is None or message["type"] == "groupchat": - rooms = self._rooms_from_message_and_room(message, room) - for r in rooms: - self.send_room_message(r["room_id"], content, **kwargs) + logging.info("putting in queue: %s" % content) + self.publish("message.outgoing.%s" % backend, e) + + def reply(self, event, content=None, message=None, package_for_scheduling=False, **kwargs): + message = self.get_message(message) + + if "channel" in kwargs: + logging.error( + "I was just asked to talk to %(channel)s, but I can't use channel using .reply() - " + "it's just for replying to the person who talked to me. Please use .say() instead." % kwargs + ) + return + if "service" in kwargs: + logging.error( + "I was just asked to talk to %(service)s, but I can't use a service using .reply() - " + "it's just for replying to the person who talked to me. Please use .say() instead." % kwargs + ) + return + if "room" in kwargs: + logging.error( + "I was just asked to talk to %(room)s, but I can't use room using .reply() - " + "it's just for replying to the person who talked to me. Please use .say() instead." % kwargs + ) + return + + # Be really smart about what we're getting back. + if ( + ( + (event and hasattr(event, "will_internal_type") and event.will_internal_type == "Message") or + (event and hasattr(event, "will_internal_type") and event.will_internal_type == "Event") + ) and type(content) == type("words") + ): + # "1.x world - user passed a message and a string. Keep rolling." + pass + elif ( + ( + (content and hasattr(content, "will_internal_type") and content.will_internal_type == "Message") or + (content and hasattr(content, "will_internal_type") and content.will_internal_type == "Event") + ) and type(event) == type("words") + ): + # "User passed the string and message object backwards, and we're in a 1.x world" + temp_content = content + content = event + event = temp_content + del temp_content + elif ( + type(event) == type("words") and + not content + ): + # "We're in the Will 2.0 automagic event finding." + content = event + event = self.message + else: - self.send_direct_message(message.sender["hipchat_id"], content, **kwargs) - - def reply(self, message, content, **kwargs): - # Valid kwargs: - # color: yellow, red, green, purple, gray, random. Default is green. - # html: Display HTML or not. Default is False - # notify: Ping everyone. Default is False - - content = self._prepared_content(content, message, kwargs) - if message is None or message["type"] == "groupchat": - # Reply, speaking to the room. - try: - content = "@%s %s" % (message.sender["nick"], content) - except TypeError: - content = "%s\nNote: I was told to reply, but this message didn't come from a person!" % (content,) - - self.say(content, message=message, **kwargs) - - elif message['type'] in ('chat', 'normal'): - # Reply to the user (1-1 chat) - - self.send_direct_message(message.sender["hipchat_id"], content, **kwargs) - - def set_topic(self, topic, message=None, room=None): - - if message is None or message["type"] == "groupchat": - rooms = self._rooms_from_message_and_room(message, room) - for r in rooms: - self.set_room_topic(r["room_id"], topic) - elif message['type'] in ('chat', 'normal'): - self.send_direct_message( - message.sender["hipchat_id"], - "I can't set the topic of a one-to-one chat. Let's just talk." + # "No magic needed." + pass + + # Be smart about backend. + if hasattr(event, "data"): + message = event.data + elif hasattr(self, "message") and hasattr(self.message, "data"): + message = self.message.data + + backend = self.get_backend(message) + if backend: + e = Event( + type="reply", + content=content, + topic="message.outgoing.%s" % backend, + source_message=message, + kwargs=kwargs, ) + if package_for_scheduling: + return e + else: + self.publish("message.outgoing.%s" % backend, e) - def schedule_say(self, content, when, message=None, room=None, *args, **kwargs): + def set_topic(self, topic, message=None, room=None, channel=None, service=None, **kwargs): + if channel: + room = channel + elif room: + channel = room + + message = self.get_message(message) + message = self._trim_for_execution(message) + backend = self.get_backend(message, service=service) + e = Event( + type="topic_change", + content=topic, + topic="message.outgoing.%s" % backend, + source_message=message, + kwargs=kwargs, + ) + self.publish("message.outgoing.%s" % backend, e) + + def schedule_say(self, content, when, message=None, room=None, channel=None, service=None, *args, **kwargs): + if channel: + room = channel + elif room: + channel = room - content = self._prepared_content(content, message, kwargs) - if message is None or message["type"] == "groupchat": - rooms = self._rooms_from_message_and_room(message, room) - for r in rooms: - self.add_room_message_to_schedule(when, content, r, *args, **kwargs) - elif message['type'] in ('chat', 'normal'): - self.add_direct_message_to_schedule(when, content, message, *args, **kwargs) + if "content" in kwargs: + if content: + del kwargs["content"] + else: + content = kwargs["content"] + + topic, packaged_event = self.say( + content, message=message, channel=channel, + service=service, package_for_scheduling=True, *args, **kwargs + ) + self.add_outgoing_event_to_schedule(when, { + "type": "message", + "topic": topic, + "event": packaged_event, + }) diff --git a/will/plugins/admin/ping.py b/will/plugins/admin/ping.py index 09130e50..96630dd1 100644 --- a/will/plugins/admin/ping.py +++ b/will/plugins/admin/ping.py @@ -1,4 +1,3 @@ -import datetime from will.plugin import WillPlugin from will.decorators import respond_to, periodic, hear, randomly, route, rendered_template, require_settings @@ -7,8 +6,8 @@ class PingPlugin(WillPlugin): @respond_to("^ping$") def ping(self, message): - self.reply(message, "PONG") + self.reply("PONG") @respond_to("^pong$") def pong(self, message): - self.reply(message, "PING") + self.reply("PING") diff --git a/will/plugins/admin/storage.py b/will/plugins/admin/storage.py index 5ed78129..df91fa05 100644 --- a/will/plugins/admin/storage.py +++ b/will/plugins/admin/storage.py @@ -4,12 +4,12 @@ class StoragePlugin(WillPlugin): - @respond_to("^How big is the db?", admin_only=True) + @respond_to("^How big is the db?", acl=["admins"]) def db_size(self, message): self.bootstrap_storage() self.say("It's %s." % self.storage.size(), message=message) - @respond_to("^SERIOUSLY. Clear (?P%s:" % k for line in help_data: if line: @@ -23,4 +28,4 @@ def help(self, message): line = " %s%s" % (line[:line.find(":")], line[line.find(":"):]) help_text += "
%s" % line - self.say(help_text, message=message, html=True) + self.say(help_text, html=True) diff --git a/will/plugins/productivity/bitly.py b/will/plugins/productivity/bitly.py index 2345e5b8..c6dffdf3 100644 --- a/will/plugins/productivity/bitly.py +++ b/will/plugins/productivity/bitly.py @@ -1,9 +1,6 @@ # coding: utf-8 -from will.utils import show_valid, error, warn, print_head - from will.plugin import WillPlugin -from will.decorators import (respond_to, periodic, hear, randomly, route, - rendered_template, require_settings) +from will.decorators import respond_to, periodic, hear, randomly, route, rendered_template, require_settings from will import settings diff --git a/will/plugins/productivity/hangout.py b/will/plugins/productivity/hangout.py index c9322411..2e5d0da0 100644 --- a/will/plugins/productivity/hangout.py +++ b/will/plugins/productivity/hangout.py @@ -1,6 +1,5 @@ from will.plugin import WillPlugin -from will.decorators import respond_to, periodic, hear, randomly, route,\ - rendered_template, require_settings +from will.decorators import respond_to, periodic, hear, randomly, route, rendered_template, require_settings from will import settings diff --git a/will/plugins/productivity/images.py b/will/plugins/productivity/images.py index 235cdd46..e3666684 100644 --- a/will/plugins/productivity/images.py +++ b/will/plugins/productivity/images.py @@ -1,5 +1,7 @@ +import logging import random import requests +from will import settings from will.plugin import WillPlugin from will.decorators import respond_to, periodic, hear, randomly, route, rendered_template, require_settings @@ -9,19 +11,104 @@ class ImagesPlugin(WillPlugin): @respond_to("image me (?P
-
- {% for user in internal_roster %}
-
- @{{user.nick|lower}} - {{user.name}}. (# {{user.hipchat_id}}) + {% for person in people %} +
- @{{person.nick|lower}} - {{person.name}}. (# {{person.hipchat_id}}) {% endfor %}