From a30b7c066d92661bb055657c520c138cdd20f53d Mon Sep 17 00:00:00 2001 From: Brian Gallew Date: Fri, 19 Mar 2021 09:43:51 +0000 Subject: [PATCH] Switch to slack_sdk and add support to ignore users and send files (#428) * Switch to slack_sdk, various extensions * Use Slack Events API * Added file upload support * Ignore users via config setting * Bump version for upcoming release * pep8 compliance * Drop support for python 2.7 * Whole lot of housekeeping Co-authored-by: Brian Gallew --- docs/config.md | 1 + will/__init__.py | 2 +- will/backends/encryption/aes.py | 66 +-- will/backends/io_adapters/base.py | 5 + will/backends/io_adapters/slack.py | 753 ++++++++++++++++----------- will/main.py | 4 +- will/plugin.py | 101 ++-- will/plugins/help/help.py | 18 +- will/plugins/help/programmer_help.py | 11 +- will/requirements/base.txt | 13 +- will/requirements/slack.txt | 6 +- will/utils.py | 2 + 12 files changed, 585 insertions(+), 397 deletions(-) diff --git a/docs/config.md b/docs/config.md index 85448a53..8b0b357d 100644 --- a/docs/config.md +++ b/docs/config.md @@ -51,6 +51,7 @@ Config.py is where all of your non-sensitive settings should go. This includes - `ENABLE_INTERNAL_ENCRYPTION`: the option to turn off internal encryption (not recommended, but you can do it.) - `PROXY_URL`: Proxy server to use, consider exporting it as `WILL_PROXY_URL` environment variable, if it contains sensitive information - `ACL`: Define arbitrary groups of users which can be used to restrict access to certain Will commands. See [access control](plugins/builtins/#access-control) for details. +- `IGNORED_USERS`: A list of Slack userids which should be ignored. Particularly useful to prevent bots from talking to each other. Slack only. - and all of your non-sensitive plugin settings. diff --git a/will/__init__.py b/will/__init__.py index 4260069c..3c00bb49 100644 --- a/will/__init__.py +++ b/will/__init__.py @@ -1 +1 @@ -VERSION = "2.1.3" +VERSION = "2.2.0" diff --git a/will/backends/encryption/aes.py b/will/backends/encryption/aes.py index 2cd31b11..70776b4f 100644 --- a/will/backends/encryption/aes.py +++ b/will/backends/encryption/aes.py @@ -1,37 +1,38 @@ +'Encrypt stored data' + import binascii -import base64 -import codecs -import dill as pickle import hashlib import logging -from Crypto.Cipher import AES -import random import os -import traceback + +import dill as pickle +from Crypto.Cipher import AES from will import settings from will.backends.encryption.base import WillBaseEncryptionBackend +# pylint: disable=no-member BS = 16 key = hashlib.sha256(settings.SECRET_KEY.encode("utf-8")).digest() -def pad(s): - s = "%s%s" % (s.decode("utf-8"), ((BS - len(s) % BS) * "~")) - return s +def pad(s: bytes) -> str: + '''Ensure the data to be encrypted has sufficient padding. + Arbitrarily adding ~ to the end, so your message better not end with ~.''' + return "%s%s" % (s.decode("utf-8"), ((BS - len(s) % BS) * "~")) -def unpad(s): - while s.endswith(str.encode("~")): - s = s[:-1] - return s +def unpad(s: bytes) -> bytes: + 'Removes all ~ on the end of the message.' + return s.rstrip(b'~') class AESEncryption(WillBaseEncryptionBackend): - - @classmethod - def encrypt_to_b64(cls, raw): + 'AES encryption backend' + @staticmethod + def encrypt_to_b64(raw): + 'encrypt and b64-encode data' try: enc = binascii.b2a_base64(pickle.dumps(raw, -1)) if settings.ENABLE_INTERNAL_ENCRYPTION: @@ -39,27 +40,28 @@ def encrypt_to_b64(cls, raw): cipher = AES.new(key, AES.MODE_CBC, iv) enc = binascii.b2a_base64(cipher.encrypt(pad(enc))) return "%s/%s" % (iv.decode("utf-8"), enc.decode("utf-8")) - else: - return enc - except: - logging.critical("Error preparing message for the wire: \n%s" % traceback.format_exc()) + return enc + except Exception: + logging.exception("Error preparing message for the wire: \n%s", raw) return None - @classmethod - def decrypt_from_b64(cls, raw_enc): + @staticmethod + def decrypt_from_b64(enc): + 'decrypt b64-encoded data' try: - if settings.ENABLE_INTERNAL_ENCRYPTION: - iv = raw_enc[:BS] - enc = raw_enc[BS+1:] + if b'/' in enc and enc.index(b'/') == BS: + iv = enc[:BS] + encrypted_data = enc[BS+1:] cipher = AES.new(key, AES.MODE_CBC, iv) - enc = unpad(cipher.decrypt(binascii.a2b_base64(enc))) - return pickle.loads(binascii.a2b_base64(enc)) + decrypted_data = unpad(cipher.decrypt(binascii.a2b_base64(encrypted_data))) + return pickle.loads(binascii.a2b_base64(decrypted_data)) except (KeyboardInterrupt, SystemExit): pass - except: - logging.warn("Error decrypting. Attempting unencrypted load to ease migration.") - return pickle.loads(binascii.a2b_base64(raw_enc)) + except Exception: + logging.debug("Error decrypting. Attempting unencrypted load to ease migration.") + return pickle.loads(binascii.a2b_base64(enc)) -def bootstrap(settings): - return AESEncryption(settings) +def bootstrap(encryption_settings): + 'Returns the encryption module' + return AESEncryption(encryption_settings) diff --git a/will/backends/io_adapters/base.py b/will/backends/io_adapters/base.py index d0a07d69..89666dbb 100644 --- a/will/backends/io_adapters/base.py +++ b/will/backends/io_adapters/base.py @@ -40,6 +40,11 @@ def normalize_incoming_event(self, event): def handle_incoming_event(self, event): try: + user_id = event.get('user') + if user_id in getattr(settings, 'IGNORED_USERS', list()): + logging.debug('Ignored %s, %s', user_id, event) + return + m = self.normalize_incoming_event(event) if m: self.pubsub.publish("message.incoming", m, reference_message=m) diff --git a/will/backends/io_adapters/slack.py b/will/backends/io_adapters/slack.py index 9dc773e0..fa4bf29d 100644 --- a/will/backends/io_adapters/slack.py +++ b/will/backends/io_adapters/slack.py @@ -1,70 +1,123 @@ +"""Slack adapter for willbot +""" +# pylint: disable=no-member import json import logging import random -import re -import requests import sys import time import traceback -from websocket import WebSocketConnectionClosedException +import urllib +from multiprocessing import Process +from threading import Thread +from typing import Dict, Optional +import slack_sdk from markdownify import MarkdownConverter +from slack_sdk.rtm import RTMClient from will import settings -from .base import IOBackend -from will.utils import Bunch, UNSURE_REPLIES, clean_for_pickling +from will.abstractions import Channel, Event, Message, Person from will.mixins import SleepMixin, StorageMixin -from multiprocessing import Process -from will.abstractions import Event, Message, Person, Channel -from slackclient import SlackClient -from slackclient.server import SlackConnectionError +from will.utils import UNSURE_REPLIES, clean_for_pickling -SLACK_SEND_URL = "https://slack.com/api/chat.postMessage" -SLACK_SET_TOPIC_URL = "https://slack.com/api/channels.setTopic" -SLACK_PRIVATE_SET_TOPIC_URL = "https://slack.com/api/groups.setTopic" +from .base import IOBackend + +# https://api.slack.com/docs/rate-limits says the actual max is 16k, but that includes the entire POST content, and +# warns about multibyte characters. Turns out the API will disconnect you if there are more than 4031 characters +# in the message. Documentation is hard. +MAX_MESSAGE_SIZE = 4031 class SlackMarkdownConverter(MarkdownConverter): + "Extended Markdown converter" - def convert_strong(self, el, text): - return '*%s*' % text if text else '' + def convert_strong(self, _, text): # pylint: disable=no-self-use + "Normal markup is incorrect for Slack" + return "*%s*" % text if text else "" def convert_a(self, el, text): - href = el.get('href') - title = el.get('title') - if self.options['autolinks'] and text == href and not title: + "dress up links for Slack" + href = el.get("href") + title = el.get("title") + if self.options["autolinks"] and text == href and not title: # Shortcut syntax - return '<%s>' % href - title_part = ' "%s"' % title.replace('"', r'\"') if title else '' - return '<%s%s|%s>' % (href, title_part, text or '') if href else text or '' + return "<%s>" % href + title_part = ' "%s"' % title.replace('"', r"\"") if title else "" + return "<%s%s|%s>" % (href, title_part, text or "") if href else text or "" -class SlackBackend(IOBackend, SleepMixin, StorageMixin): +class SlackBackend( + IOBackend, SleepMixin, StorageMixin +): # pylint: disable=too-many-instance-attributes + "Adapter that lets Will talk to Slack" friendly_name = "Slack" internal_name = "will.backends.io_adapters.slack" required_settings = [ { "name": "SLACK_API_TOKEN", - "obtain_at": """1. Go to https://api.slack.com/custom-integrations/legacy-tokens and sign in as yourself (or a user for Will). -2. Find the workspace you want to use, and click "Create token." -3. Set this token as SLACK_API_TOKEN.""" + "obtain_at": "1. Go to https://api.slack.com/custom-integrations/legacy-tokens" + " and sign in as yourself (or a user for Will).\n" + '2. Find the workspace you want to use, and click "Create token.\n' + "3. Set this token as SLACK_API_TOKEN.", } ] + PAGE_LIMIT = 1000 + _channels: Dict[str, Channel] = dict() + _people: Dict[str, Person] = dict() + _default_channel = None + rtm_thread: Optional[RTMClient] = None + complained_about_default = False + complained_uninvited = False + _client = None + me = None + handle = None def get_channel_from_name(self, name): - for k, c in self.channels.items(): + "Decodes human-readable name into the Slack channel name" + for c in self.channels.values(): if c.name.lower() == name.lower() or c.id.lower() == name.lower(): return c + webclient = self.client._web_client # pylint: disable=protected-access + try: + channel_info = webclient.conversations_info(channel=name)["channel"] + except slack_sdk.errors.SlackApiError: + logging.warning('Error looking up Slack channel %s', name) + return None + channel_members = webclient.conversations_members(channel=name) + now = time.time() + members = { + x: self.people[x] + for x in channel_members.get("members", list()) + if x in self.people + } + logging.debug("took %0.2f seconds to scan people data", time.time() - now) + return Channel( + name=channel_info.get("name", name), + id=channel_info["id"], + members=members, + source=name, + is_channel=False, + is_group=False, + is_im=channel_info["is_im"], + is_private=True, + ) def normalize_incoming_event(self, event): - + "Makes a Slack event look like all the other events we handle" + event_type = event.get("type") + event_subtype = event.get("subtype") + logging.debug("event type: %s, subtype: %s", event_type, event_subtype) if ( - "type" in event - and event["type"] == "message" - and ("subtype" not in event or event["subtype"] != "message_changed") + ( + event_subtype is None + and event_type not in ["message_changed", "message.incoming"] + ) # Ignore thread summary events (for now.) - # TODO: We should stack these into the history. - and ("subtype" not in event or ("message" in event and "thread_ts" not in event["message"])) + and ( + event_subtype is None + or ("message" in event and "thread_ts" not in event["message"]) + ) ): # print("slack: normalize_incoming_event - %s" % event) # Sample of group message @@ -87,51 +140,44 @@ def normalize_incoming_event(self, event): # u'user': u'U5ACF70RH', u'ts': u'1507601477.000063'}], # u'type': u'message', u'bot_id': u'B5HL9ABFE'}, # u'type': u'message', u'hidden': True, u'channel': u'D5HGP0YE7'} - + logging.debug("we like that event!") sender = self.people[event["user"]] - channel = clean_for_pickling(self.channels[event["channel"]]) + channel = self.get_channel_from_name(event["channel"]) + is_private_chat = getattr(channel, "is_private", False) + is_direct = getattr(getattr(channel, "source", None), 'is_im', False) + channel = clean_for_pickling(channel) # print "channel: %s" % channel interpolated_handle = "<@%s>" % self.me.id real_handle = "@%s" % self.me.handle will_is_mentioned = False will_said_it = False - is_private_chat = False - thread = None if "thread_ts" in event: thread = event["thread_ts"] - # If the parent thread is a 1-1 between Will and I, also treat that as direct. - # Since members[] still comes in on the thread event, we can trust this, even if we're - # in a thread. - if channel.id == channel.name: - is_private_chat = True - - # <@U5GUL9D9N> hi - # TODO: if there's a thread with just will and I on it, treat that as direct. - is_direct = False - if is_private_chat or event["text"].startswith(interpolated_handle) or event["text"].startswith(real_handle): - is_direct = True + if interpolated_handle in event["text"] or real_handle in event["text"]: + will_is_mentioned = True if event["text"].startswith(interpolated_handle): - event["text"] = event["text"][len(interpolated_handle):].strip() + event["text"] = event["text"][len(interpolated_handle):] if event["text"].startswith(real_handle): - event["text"] = event["text"][len(real_handle):].strip() + event["text"] = event["text"][len(real_handle):] - if interpolated_handle in event["text"] or real_handle in event["text"]: - will_is_mentioned = True + # sometimes autocomplete adds a : to the usename, but it's certainly extraneous. + if will_is_mentioned and event["text"][0] == ":": + event["text"] = event["text"][1:] if event["user"] == self.me.id: will_said_it = True m = Message( - content=event["text"], - type=event["type"], - is_direct=is_direct, + content=event["text"].strip(), + type=event_type, + is_direct=is_direct or will_is_mentioned, is_private_chat=is_private_chat, - is_group_chat=not is_private_chat, + is_group_chat=not (is_private_chat or is_direct), backend=self.internal_name, sender=sender, channel=channel, @@ -142,99 +188,123 @@ def normalize_incoming_event(self, event): original_incoming_event=clean_for_pickling(event), ) return m - else: - # An event type the slack ba has no idea how to handle. - pass + # An event type the slack ba has no idea how to handle. + return None def set_topic(self, event): - headers = {'Accept': 'text/plain'} + """Sets the channel topic. This doesn't actually work anymore since Slack has removed that ability from bots. + Leaving the code here in case they re-enable it.""" data = self.set_data_channel_and_thread(event) - data.update({ - "token": settings.SLACK_API_TOKEN, - "as_user": True, - "topic": event.content, - }) - if data["channel"].startswith("G"): - url = SLACK_PRIVATE_SET_TOPIC_URL - else: - url = SLACK_SET_TOPIC_URL - r = requests.post( - url, - headers=headers, - data=data, - **settings.REQUESTS_OPTIONS + self.client._web_client.api_call( # pylint: disable=protected-access + "conversations.setTopic", + topic=event.content, + channel=data["channel"], ) - self.handle_request(r, data) - - def handle_outgoing_event(self, event): - if event.type in ["say", "reply"]: - if "kwargs" in event and "html" in event.kwargs and event.kwargs["html"]: - event.content = SlackMarkdownConverter().convert(event.content) - - event.content = event.content.replace("&", "&") - event.content = event.content.replace(r"\_", "_") - kwargs = {} - if "kwargs" in event: - kwargs.update(**event.kwargs) - - if hasattr(event, "source_message") and event.source_message and "channel" not in kwargs: - self.send_message(event) - else: - # Came from webhook/etc - # TODO: finish this. - target_channel = kwargs.get("room", kwargs.get("channel", None)) - if target_channel: - event.channel = self.get_channel_from_name(target_channel) - if event.channel: + def send_file(self, event): + 'Sometimes you need to upload an image or file' + + try: + logging.info('EVENT: %s', str(event)) + data = { + 'filename': event.filename, + 'filetype': event.filetype + } + self.set_data_channel_and_thread(event, data) + # This is just *silly* + if 'thread_ts' in data: + del data['thread_ts'] + data['channels'] = data['channel'] + del data['channel'] + + logging.debug('calling files_uploads with: %s', data) + result = self.client._web_client.api_call( # pylint: disable=protected-access + 'files.upload', files={'file': event.file}, params=data) + logging.debug('send_file result: %s', result) + except Exception: + logging.exception("Error in send_file handling %s", event) + + def handle_outgoing_event(self, event, retry=5): + "Process an outgoing event" + try: + if event.type in ["say", "reply"]: + if "kwargs" in event and "html" in event.kwargs and event.kwargs["html"]: + event.content = SlackMarkdownConverter().convert(event.content) + + event.content = event.content.replace("&", "&") + event.content = event.content.replace(r"\_", "_") + + kwargs = {} + if "kwargs" in event: + kwargs.update(**event.kwargs) + + if kwargs.get('update', None) is not None: + self.update_message(event) + elif ( + hasattr(event, "source_message") + and event.source_message + and "channel" not in kwargs + ): + self.send_message(event) + else: + # Came from webhook/etc + target_channel = kwargs.get("room", kwargs.get("channel", None)) + if target_channel: + event.channel = self.get_channel_from_name(target_channel) + if event.channel: + self.send_message(event) + else: + logging.error( + "I was asked to post to the slack %s channel, but it doesn't exist.", + target_channel, + ) + if self.default_channel: + event.channel = self.get_channel_from_name( + self.default_channel + ) + event.content = ( + event.content + " (for #%s)" % target_channel + ) + self.send_message(event) + + elif self.default_channel: + event.channel = self.get_channel_from_name(self.default_channel) self.send_message(event) else: - logging.error( - "I was asked to post to the slack %s channel, but it doesn't exist.", - target_channel + logging.critical( + "I was asked to post to a slack default channel, but I'm nowhere." + "Please invite me somewhere with '/invite @%s'", + self.me.handle, ) - if self.default_channel: - event.channel = self.get_channel_from_name(self.default_channel) - event.content = event.content + " (for #%s)" % target_channel - self.send_message(event) - - elif self.default_channel: - event.channel = self.get_channel_from_name(self.default_channel) - self.send_message(event) - else: - logging.critical( - "I was asked to post to a slack default channel, but I'm nowhere." - "Please invite me somewhere with '/invite @%s'", self.me.handle - ) - - if event.type in ["topic_change", ]: - self.set_topic(event) - elif ( - event.type == "message.no_response" - and event.data.is_direct - and event.data.will_said_it is False - ): - event.content = random.choice(UNSURE_REPLIES) - self.send_message(event) - - def handle_request(self, r, data): - resp_json = r.json() - if not resp_json["ok"]: - if resp_json["error"] == "not_in_channel": - channel = self.get_channel_from_name(data["channel"]) - if not hasattr(self, "me") or not hasattr(self.me, "handle"): - self.people - - logging.critical( - "I was asked to post to the slack %s channel, but I haven't been invited. " - "Please invite me with '/invite @%s'" % (channel.name, self.me.handle) - ) - else: - logging.error("Error sending to slack: %s" % resp_json["error"]) - logging.error(resp_json) - assert resp_json["ok"] - def set_data_channel_and_thread(self, event, data={}): + elif event.type in [ + "topic_change", + ]: + self.set_topic(event) + elif event.type in [ + "file.upload", + ]: + self.send_file(event) + elif ( + event.type == "message.no_response" + and event.data.is_direct + and event.data.will_said_it is False + ): + event.content = random.choice(UNSURE_REPLIES) + self.send_message(event) + except (urllib.error.URLError, Exception): # websocket got closed, no idea + if retry < 1: + sys.exit() + del self._client + time.sleep(5) # pause 5 seconds just because + self.client + self.handle_outgoing_event(event, retry=retry-1) + + @staticmethod + def set_data_channel_and_thread(event, data=None): + "Update data with the channel/thread information from event" + if data is None: + data = dict() if "channel" in event: # We're coming off an explicit set. channel_id = event.channel.id @@ -244,16 +314,12 @@ def set_data_channel_and_thread(self, event, data={}): if hasattr(event.source_message, "data"): channel_id = event.source_message.data.channel.id if hasattr(event.source_message.data, "thread"): - data.update({ - "thread_ts": event.source_message.data.thread - }) + data.update({"thread_ts": event.source_message.data.thread}) else: # Mentions that come back via self.say() with a specific room (I think) channel_id = event.source_message.channel.id if hasattr(event.source_message, "thread"): - data.update({ - "thread_ts": event.source_message.thread - }) + data.update({"thread_ts": event.source_message.thread}) else: # Mentions that come back via self.reply() if hasattr(event.data, "original_incoming_event"): @@ -262,21 +328,33 @@ def set_data_channel_and_thread(self, event, data={}): else: channel_id = event.data.original_incoming_event.channel else: - if hasattr(event.data["original_incoming_event"].data.channel, "id"): - channel_id = event.data["original_incoming_event"].data.channel.id + if hasattr( + event.data["original_incoming_event"].data.channel, "id" + ): + channel_id = event.data[ + "original_incoming_event" + ].data.channel.id else: channel_id = event.data["original_incoming_event"].data.channel try: # If we're starting a thread - if "kwargs" in event and "start_thread" in event.kwargs and event.kwargs["start_thread"] and ("thread_ts" not in data or not data["thread_ts"]): + if ( + "kwargs" in event + and event.kwargs.get("start_thread", False) + and ("thread_ts" not in data or not data["thread_ts"]) + ): if hasattr(event.source_message, "original_incoming_event"): - data.update({ - "thread_ts": event.source_message.original_incoming_event["ts"] - }) + data.update( + { + "thread_ts": event.source_message.original_incoming_event["ts"] + } + ) elif ( hasattr(event.source_message, "data") - and hasattr(event.source_message.data, "original_incoming_event") + and hasattr( + event.source_message.data, "original_incoming_event" + ) and "ts" in event.source_message.data.original_incoming_event ): logging.error( @@ -285,58 +363,74 @@ def set_data_channel_and_thread(self, event, data={}): "used .say() and threading off of your message.\n" "Please update your plugin to use .reply() when you have a second!" ) - data.update({ - "thread_ts": event.source_message.data.original_incoming_event["ts"] - }) + data.update( + { + "thread_ts": event.source_message.data.original_incoming_event[ + "ts" + ] + } + ) else: if hasattr(event.data.original_incoming_event, "thread_ts"): - data.update({ - "thread_ts": event.data.original_incoming_event.thread_ts - }) + data.update( + {"thread_ts": event.data.original_incoming_event.thread_ts} + ) elif "thread" in event.data.original_incoming_event.data: - data.update({ - "thread_ts": event.data.original_incoming_event.data.thread - }) - except: + data.update( + { + "thread_ts": event.data.original_incoming_event.data.thread + } + ) + except Exception: logging.info(traceback.format_exc().split(" ")[-1]) - pass - data.update({ - "channel": channel_id, - }) + + data.update( + { + "channel": channel_id, + } + ) return data - def send_message(self, event): - if event.content == '' or event.content is None: - # slack errors with no_text if empty message - return + def get_event_data(self, event): + "Send a Slack message" data = {} if hasattr(event, "kwargs"): data.update(event.kwargs) # Add slack-specific functionality if "color" in event.kwargs: - data.update({ - "attachments": json.dumps([ - { - "fallback": event.content, - "color": self._map_color(event.kwargs["color"]), - "text": event.content, - } - ]), - }) + data.update( + { + "attachments": json.dumps( + [ + { + "fallback": event.content, + "color": self._map_color(event.kwargs["color"]), + "text": event.content, + } + ] + ), + } + ) elif "attachments" in event.kwargs: - data.update({ - "text": event.content, - "attachments": json.dumps(event.kwargs["attachments"]) - }) + data.update( + { + "text": event.content, + "attachments": json.dumps(event.kwargs["attachments"]), + } + ) else: - data.update({ - "text": event.content, - }) + data.update( + { + "text": event.content, + } + ) else: - data.update({ - "text": event.content, - }) + data.update( + { + "text": event.content, + } + ) data = self.set_data_channel_and_thread(event, data=data) @@ -345,78 +439,136 @@ def send_message(self, event): if data["text"].find("<@") != -1: data["text"] = data["text"].replace("<@", "<@") data["text"] = data["text"].replace(">", ">") + if len(data['text']) > MAX_MESSAGE_SIZE: + new_event = Event( + type='file.upload', + # Removes "code" markers from around the item and then makes it bytes + file=data['text'].strip('```').encode('utf-8'), + filename=getattr(event, 'filename', getattr(event, 'title', 'response')), + filetype=getattr(event, 'filetype', 'text'), + source_message=event.source_message, + kwargs=event.kwargs, + ) + try: + self.send_file(new_event) + except Exception: + logging.exception('Error sending file') + return None elif "attachments" in data and "text" in data["attachments"][0]: if data["attachments"][0]["text"].find("<@") != -1: - data["attachments"][0]["text"] = data["attachments"][0]["text"].replace("<@", "<@") - data["attachments"][0]["text"] = data["attachments"][0]["text"].replace(">", ">") + data["attachments"][0]["text"] = data["attachments"][0]["text"].replace( + "<@", "<@" + ) + data["attachments"][0]["text"] = data["attachments"][0]["text"].replace( + ">", ">" + ) - data.update({ - "token": settings.SLACK_API_TOKEN, - "as_user": True, - }) + data.update( + { + "token": settings.SLACK_API_TOKEN, + "as_user": True, + } + ) if hasattr(event, "kwargs") and "html" in event.kwargs and event.kwargs["html"]: - data.update({ - "parse": "none", - }) - - headers = {'Accept': 'text/plain'} - r = requests.post( - SLACK_SEND_URL, - headers=headers, - data=data, - **settings.REQUESTS_OPTIONS + data.update( + { + "parse": "none", + } + ) + return data + + def send_message(self, event): + "Send a Slack message" + if event.content == "" or event.content is None: + # slack errors with no_text if empty message + return + data = self.get_event_data(event) + if data is None: + return + self.client._web_client.api_call( # pylint: disable=protected-access + "chat.postMessage", data=data ) - self.handle_request(r, data) - def _map_color(self, color): - # Turn colors into hex values, handling old slack colors, etc + def update_message(self, event): + "Update a Slack message" + if event.content == "" or event.content is None: + # slack errors with no_text if empty message + return + data = self.get_event_data(event) + if data is None: + return + redis_key = f'slack_update_cache_{data["update"]}' + if not hasattr(self, 'storage'): + self.bootstrap_storage() + timestamp = self.storage.redis.get(redis_key) + if not timestamp: + result = self.client._web_client.api_call( # pylint: disable=protected-access + "chat.postMessage", data=data + ) + if result.get('ts', None): + self.storage.redis.set(redis_key, result['ts'], ex=3600) + else: + logging.error('Failure sending %s: %s', event, result.get('error', 'Unknown error')) + else: + data['ts'] = timestamp + result = self.client._web_client.api_call( # pylint: disable=protected-access + "chat.update", data=data + ) + if result.get('ok', False) is False: + logging.error('Failure updating %s: %s', event, result.get('error', 'Unknown error')) + + @staticmethod + def _map_color(color): + "Turn colors into hex values, handling old slack colors, etc" if color == "red": return "danger" - elif color == "yellow": + if color == "yellow": return "warning" - elif color == "green": + if color == "green": return "good" - return color def join_channel(self, channel_id): - return self.client.api_call( + "Join a channel" + return self.client._web_client.api_call( # pylint: disable=protected-access "channels.join", channel=channel_id, ) @property def people(self): - if not hasattr(self, "_people") or self._people is {}: + "References/initializes our internal people cache" + if not self._people: self._update_people() return self._people @property def default_channel(self): - if not hasattr(self, "_default_channel") or not self._default_channel: + "References/initializes our default channel" + if not self._default_channel: self._decide_default_channel() return self._default_channel @property def channels(self): - if not hasattr(self, "_channels") or self._channels is {}: + "References/initializes our internal channel cache" + if not self._channels: self._update_channels() return self._channels @property def client(self): - if not hasattr(self, "_client"): - self._client = SlackClient(settings.SLACK_API_TOKEN) + "References/initializes our RTM client" + if self._client is None: + self._client = RTMClient( + token=settings.SLACK_API_TOKEN, run_async=False, auto_reconnect=True + ) return self._client def _decide_default_channel(self): + "Selects a default channel" self._default_channel = None - if not hasattr(self, "complained_about_default"): - self.complained_about_default = False - self.complained_uninvited = False - - # Set self.me - self.people + self.people # Set self.me # pylint: disable=pointless-statement if hasattr(settings, "SLACK_DEFAULT_CHANNEL"): channel = self.get_channel_from_name(settings.SLACK_DEFAULT_CHANNEL) @@ -426,8 +578,10 @@ def _decide_default_channel(self): return elif not self.complained_about_default: self.complained_about_default = True - logging.error("The defined default channel(%s) does not exist!", - settings.SLACK_DEFAULT_CHANNEL) + logging.error( + "The defined default channel(%s) does not exist!", + settings.SLACK_DEFAULT_CHANNEL, + ) for c in self.channels.values(): if c.name != c.id and self.me.id in c.members: @@ -436,19 +590,27 @@ def _decide_default_channel(self): self.complained_uninvited = True logging.critical("No channels with me invited! No messages will be sent!") - def _update_channels(self): + def _update_channels(self, client=None): + "Updates our internal list of channels. Kind of expensive." channels = {} - for c in self.client.server.channels: - members = {} - for m in c.members: - members[m] = self.people[m] - - channels[c.id] = Channel( - id=c.id, - name=c.name, - source=clean_for_pickling(c), - members=members - ) + if client: + for page in client.conversations_list( + limit=self.PAGE_LIMIT, + exclude_archived=True, + types="public_channel,private_channel,mpim,im", + ): + for channel in page["channels"]: + members = {} + for m in channel.get("members", list()): + if m in self.people: + members[m] = self.people[m] + + channels[channel["id"]] = Channel( + id=channel["id"], + name=channel.get("name", channel["id"]), + source=clean_for_pickling(channel), + members=members, + ) if len(channels.keys()) == 0: # Server isn't set up yet, and we're likely in a processing thread, if self.load("slack_channel_cache", None): @@ -457,41 +619,34 @@ def _update_channels(self): self._channels = channels self.save("slack_channel_cache", channels) - def _update_people(self): + def _update_people(self, client=None): + "Updates our internal list of Slack users. Kind of expensive." people = {} - - self.handle = self.client.server.username - - for k, v in self.client.server.users.items(): - user_timezone = None - if v.tz: - user_timezone = v.tz - people[k] = Person( - id=v.id, - mention_handle="<@%s>" % v.id, - handle=v.name, - source=clean_for_pickling(v), - name=v.real_name, - ) - if v.name == self.handle: - self.me = Person( - id=v.id, - mention_handle="<@%s>" % v.id, - handle=v.name, - source=clean_for_pickling(v), - name=v.real_name, - ) - if user_timezone and user_timezone != 'unknown': - people[k].timezone = user_timezone - if v.name == self.handle: - self.me.timezone = user_timezone + if client: + for page in client.users_list(limit=self.PAGE_LIMIT): + for member in page["members"]: + if member["deleted"]: + continue + member_id = member["id"] + user_timezone = member.get("tz") + people[member_id] = Person( + id=member_id, + mention_handle=member.get("mention_handle", ""), + handle=member["name"], + source=clean_for_pickling(member), + name=member.get("real_name", ""), + ) + if member["name"] == self.handle: + self.me = people[member_id] + if user_timezone and user_timezone != "unknown": + people[member_id].timezone = user_timezone if len(people.keys()) == 0: # Server isn't set up yet, and we're likely in a processing thread, if self.load("slack_people_cache", None): self._people = self.load("slack_people_cache", None) - if not hasattr(self, "me") or not self.me: + if self.me is None: self.me = self.load("slack_me_cache", None) - if not hasattr(self, "handle") or not self.handle: + if self.handle is None: self.handle = self.load("slack_handle_cache", None) else: self._people = people @@ -499,43 +654,37 @@ def _update_people(self): self.save("slack_me_cache", self.me) self.save("slack_handle_cache", self.handle) - def _update_backend_metadata(self): - self._update_people() - self._update_channels() + def _update_backend_metadata(self, **event): + "Updates all our internal caches. Very expenseive" + logging.debug("updating backend on event: %s", event) + name = event.get("data", dict()).get("self", dict()).get("name") + if name is not None: + self.__class__.handle = name + Thread( + target=self._update_people, args=(event["web_client"],), daemon=True + ).start() + Thread( + target=self._update_channels, args=(event["web_client"],), daemon=True + ).start() def _watch_slack_rtm(self): - while True: - try: - if self.client.rtm_connect(auto_reconnect=True): - self._update_backend_metadata() - - num_polls_between_updates = 30 / settings.EVENT_LOOP_INTERVAL # Every 30 seconds - current_poll_count = 0 - while True: - events = self.client.rtm_read() - if len(events) > 0: - # TODO: only handle events that are new. - # print(len(events)) - for e in events: - self.handle_incoming_event(e) - - # Update channels/people/me/etc every 10s or so. - current_poll_count += 1 - if current_poll_count > num_polls_between_updates: - self._update_backend_metadata() - current_poll_count = 0 - - self.sleep_for_event_loop() - except (WebSocketConnectionClosedException, SlackConnectionError): - logging.error('Encountered connection error attempting reconnect in 2 seconds') - time.sleep(2) - except (KeyboardInterrupt, SystemExit): - break - except: - logging.critical("Error in watching slack RTM: \n%s" % traceback.format_exc()) - break + "This is our main loop." + # The decorators don't work on unbound methods. Sigh. + # These are all events that should spark an update of our inventory + RTMClient.run_on(event="open")(self._update_backend_metadata) + RTMClient.run_on(event="channel_added")(self._update_backend_metadata) + RTMClient.run_on(event="user_changed")(self._update_backend_metadata) + # This just handles messages + RTMClient.run_on(event="message")(self.handle_incoming_slack_event) + self.client.start() + + def handle_incoming_slack_event(self, **kwargs): + "Event handler" + logging.debug("Handling incoming event: %s", kwargs) + self.handle_incoming_event(kwargs["data"]) def bootstrap(self): + "This is Wills Process entry point for connecting to a backend" # Bootstrap must provide a way to to have: # a) self.normalize_incoming_event fired, or incoming events put into self.incoming_queue # b) any necessary threads running for a) @@ -546,14 +695,12 @@ def bootstrap(self): # f) A way for self.handle, self.me, self.people, and self.channels to be kept accurate, # with a maximum lag of 60 seconds. - # Property, auto-inits. - self.client - - self.rtm_thread = Process(target=self._watch_slack_rtm) + self.rtm_thread = Process(target=self._watch_slack_rtm, daemon=False) self.rtm_thread.start() def terminate(self): - if hasattr(self, "rtm_thread"): + "Exit gracefully" + if self.rtm_thread is not None: self.rtm_thread.terminate() while self.rtm_thread.is_alive(): time.sleep(0.2) diff --git a/will/main.py b/will/main.py index c42b1b45..68078172 100644 --- a/will/main.py +++ b/will/main.py @@ -1079,13 +1079,13 @@ def bootstrap_plugins(self): # puts("- %s" % function_name) self.bottle_routes.append((plugin_info["class"], function_name)) - except Exception: + except Exception as e: error(plugin_name) self.startup_error( "Error bootstrapping %s.%s" % ( plugin_info["class"], function_name, - ) + ), e ) if len(plugin_warnings) > 0: show_invalid(plugin_name) diff --git a/will/plugin.py b/will/plugin.py index d6f93bea..438229af 100644 --- a/will/plugin.py +++ b/will/plugin.py @@ -1,3 +1,4 @@ +'Standard Will plugin functionality' import re import logging @@ -6,14 +7,15 @@ from will import settings from will.abstractions import Event, Message # Backwards compatability with 1.x, eventually to be deprecated. -from will.backends.io_adapters.hipchat import HipChatRosterMixin, HipChatRoomMixin from will.mixins import NaturalTimeMixin, ScheduleMixin, StorageMixin, SettingsMixin, \ EmailMixin, PubSubMixin -from will.utils import html_to_text +FILENAME_CLEANER = re.compile(r'[^-_0-9a-zA-Z]+') -class WillPlugin(EmailMixin, StorageMixin, NaturalTimeMixin, HipChatRoomMixin, HipChatRosterMixin, + +class WillPlugin(EmailMixin, StorageMixin, NaturalTimeMixin, ScheduleMixin, SettingsMixin, PubSubMixin): + 'Basic things needed by all plugins' is_will_plugin = True request = request @@ -25,13 +27,15 @@ def __init__(self, *args, **kwargs): self.message = kwargs["message"] del kwargs["message"] - super(WillPlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) - def _prepared_content(self, content, message, kwargs): + @staticmethod + def _prepared_content(content, _, __): content = re.sub(r'>\s+<', '><', content) return content - def _trim_for_execution(self, message): + @staticmethod + def _trim_for_execution(message): # Trim it down if hasattr(message, "analysis"): message.analysis = None @@ -39,10 +43,12 @@ def _trim_for_execution(self, message): message.source_message.analysis = None return message - def get_backend(self, message, service=None): + @staticmethod + def get_backend(message: Message, service: str = None): + 'Select the correct backend (module path)' backend = False if service: - for b in settings.IO_BACKENDS: + for b in settings.IO_BACKENDS: # pylint: disable=no-member if service in b: return b @@ -51,23 +57,23 @@ def get_backend(self, message, service=None): elif message and hasattr(message, "data") and hasattr(message.data, "backend"): backend = message.data.backend else: - backend = settings.DEFAULT_BACKEND + backend = settings.DEFAULT_BACKEND # pylint: disable=no-member return backend def get_message(self, message_passed): + 'Try to find a message' if not message_passed and hasattr(self, "message"): return self.message return message_passed def say(self, content, message=None, room=None, channel=None, service=None, package_for_scheduling=False, **kwargs): - logging.info("self.say") - logging.info(content) + 'Publish an event to be shipped to one of the IO backends.' if channel: room = channel elif room: channel = room - if not "channel" in kwargs and channel: + if "channel" not in kwargs and channel: kwargs["channel"] = channel message = self.get_message(message) @@ -83,38 +89,69 @@ def say(self, content, message=None, room=None, channel=None, service=None, pack ) if package_for_scheduling: return "message.outgoing.%s" % backend, e - else: - logging.info("putting in queue: %s" % content) - self.publish("message.outgoing.%s" % backend, e) + logging.info("putting in queue: %s", content) + self.publish("message.outgoing.%s" % backend, e) + return None + + def send_file(self, message, content, filename, text=None, filetype='text', + service=None, room=None, channel=None, **kwargs): + 'Sometimes you need to upload an image or file' + if channel: + room = channel + elif room: + channel = room + + if "channel" not in kwargs and channel: + kwargs["channel"] = channel + + message = self.get_message(message) + message = self._trim_for_execution(message) + backend = self.get_backend(message, service=service) + + if backend: + if isinstance(content, str): + content = content.encode('utf-8') + e = Event( + type="file.upload", + file=content, + filename=FILENAME_CLEANER.sub('_', filename), + filetype=filetype, + source_message=message, + title=text, + kwargs=kwargs, + ) + logging.info("putting in queue: %s", content) + self.publish("message.outgoing.%s" % backend, e) def reply(self, event, content=None, message=None, package_for_scheduling=False, **kwargs): + 'Publish an event to be shipped to one of the IO backends.' message = self.get_message(message) if "channel" in kwargs: logging.error( "I was just asked to talk to %(channel)s, but I can't use channel using .reply() - " - "it's just for replying to the person who talked to me. Please use .say() instead." % kwargs + "it's just for replying to the person who talked to me. Please use .say() instead.", kwargs ) - return + return None if "service" in kwargs: logging.error( "I was just asked to talk to %(service)s, but I can't use a service using .reply() - " - "it's just for replying to the person who talked to me. Please use .say() instead." % kwargs + "it's just for replying to the person who talked to me. Please use .say() instead.", kwargs ) - return + return None if "room" in kwargs: logging.error( "I was just asked to talk to %(room)s, but I can't use room using .reply() - " - "it's just for replying to the person who talked to me. Please use .say() instead." % kwargs + "it's just for replying to the person who talked to me. Please use .say() instead.", kwargs ) - return + return None # Be really smart about what we're getting back. if ( ( (event and hasattr(event, "will_internal_type") and event.will_internal_type == "Message") or (event and hasattr(event, "will_internal_type") and event.will_internal_type == "Event") - ) and type(content) == type("words") + ) and isinstance(content, str) ): # "1.x world - user passed a message and a string. Keep rolling." pass @@ -122,15 +159,12 @@ def reply(self, event, content=None, message=None, package_for_scheduling=False, ( (content and hasattr(content, "will_internal_type") and content.will_internal_type == "Message") or (content and hasattr(content, "will_internal_type") and content.will_internal_type == "Event") - ) and type(event) == type("words") + ) and isinstance(event, str) ): # "User passed the string and message object backwards, and we're in a 1.x world" - temp_content = content - content = event - event = temp_content - del temp_content + content, event = event, content elif ( - type(event) == type("words") + isinstance(event, str) and not content ): # "We're in the Will 2.0 automagic event finding." @@ -158,10 +192,11 @@ def reply(self, event, content=None, message=None, package_for_scheduling=False, ) if package_for_scheduling: return e - else: - self.publish("message.outgoing.%s" % backend, e) + self.publish("message.outgoing.%s" % backend, e) + return None def set_topic(self, topic, message=None, room=None, channel=None, service=None, **kwargs): + 'Try to set the room/channel topic.' if channel: room = channel elif room: @@ -180,10 +215,8 @@ def set_topic(self, topic, message=None, room=None, channel=None, service=None, self.publish("message.outgoing.%s" % backend, e) def schedule_say(self, content, when, message=None, room=None, channel=None, service=None, *args, **kwargs): - if channel: - room = channel - elif room: - channel = room + 'Publish an event to for the future' + channel = channel or room # This does not look like testable code, or something that will ever # happen. If we have a required "content" positional argument, if diff --git a/will/plugins/help/help.py b/will/plugins/help/help.py index df9d50db..2465a813 100644 --- a/will/plugins/help/help.py +++ b/will/plugins/help/help.py @@ -7,25 +7,23 @@ class HelpPlugin(WillPlugin): @respond_to("^help(?: (?P.*))?$") def help(self, message, plugin=None): """help: the normal help you're reading.""" - # help_data = self.load("help_files") selected_modules = help_modules = self.load("help_modules") - self.say("Sure thing, %s." % message.sender.handle) + self.say(f"Sure thing, {message.sender.handle}.", message, start_thread=True) help_text = "Here's what I know how to do:" if plugin and plugin in help_modules: - help_text = "Here's what I know how to do about %s:" % plugin + help_text = f"Here's what I know how to do about {plugin}:" selected_modules = dict() selected_modules[plugin] = help_modules[plugin] + self.say(help_text, start_thread=True) for k in sorted(selected_modules, key=lambda x: x[0]): help_data = selected_modules[k] if help_data: - help_text += "

%s:" % k + help_text = [f'```{k}'] for line in help_data: - if line: - if ":" in line: - line = "  %s%s" % (line[:line.find(":")], line[line.find(":"):]) - help_text += "
%s" % line - - self.say(help_text, html=True) + if line and ":" in line: + key, value = line.split(':', 1) + help_text.append(f" {key}: {value}") + self.say('\n'.join(help_text) + '```', message, start_thread=True) diff --git a/will/plugins/help/programmer_help.py b/will/plugins/help/programmer_help.py index 27eb4a43..38274457 100644 --- a/will/plugins/help/programmer_help.py +++ b/will/plugins/help/programmer_help.py @@ -1,6 +1,8 @@ from will.plugin import WillPlugin from will.decorators import respond_to, periodic, hear, randomly, route, rendered_template, require_settings +MAX_LINES = 18 + class ProgrammerHelpPlugin(WillPlugin): @@ -8,8 +10,7 @@ class ProgrammerHelpPlugin(WillPlugin): def help(self, message): """programmer help: Advanced programmer-y help.""" all_regexes = self.load("all_listener_regexes") - help_text = "Here's everything I know how to listen to:" - for r in all_regexes: - help_text += "\n%s" % r - - self.say(help_text, message=message) + self.say("Here's everything I know how to listen to:", message, start_thread=True) + for r in range(0, len(all_regexes), MAX_LINES): + text = "\n".join(all_regexes[r:r+MAX_LINES]) + self.say(f'```{text}```', message, start_thread=True) diff --git a/will/requirements/base.txt b/will/requirements/base.txt index 69a4f9b9..60754139 100644 --- a/will/requirements/base.txt +++ b/will/requirements/base.txt @@ -2,8 +2,8 @@ APScheduler==2.1.2 beautifulsoup4==4.6.0 bottle==0.12.7 CherryPy==3.6.0 -clint==0.3.7 -dill==0.2.1 +clint==0.5.1 +dill==0.3.3 dnspython==1.15.0 fuzzywuzzy==0.15.1 Jinja2==2.7.3 @@ -13,7 +13,7 @@ MarkupSafe==0.23 # natural==0.2.1 will-natural==0.2.1.1 parsedatetime==1.1.2 -python-Levenshtein==0.12.0 +python-Levenshtein==0.12.1 pyasn1-modules==0.0.5 pyasn1==0.1.7 pycrypto==2.6.1 @@ -22,7 +22,6 @@ pytz==2017.2 PyYAML==3.13 regex==2017.9.23 redis==2.10.6 -requests==2.20.0 -six==1.10.0 -urllib3==1.24.3 -websocket-client==0.44.0 +requests==2.25.0 +six==1.15.0 +urllib3==1.25.10 diff --git a/will/requirements/slack.txt b/will/requirements/slack.txt index 8b95fd83..6a56658a 100644 --- a/will/requirements/slack.txt +++ b/will/requirements/slack.txt @@ -1,4 +1,4 @@ -r base.txt -slackclient>=1.2.1,<1.3.0 -markdownify==0.4.1 - +slack-sdk==3.0.0 +markdownify==0.5.3 +aiohttp diff --git a/will/utils.py b/will/utils.py index 6fbbcb85..ab78dcd7 100644 --- a/will/utils.py +++ b/will/utils.py @@ -20,6 +20,8 @@ "server", "send_message", "_updatedAt", + "rtm_client", # slack-specific + "web_client" # slack-specific ]