From 287b7229623d1f7779d6b521d47c6098fa46bfdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Tue, 21 Jun 2016 06:41:38 +0300 Subject: [PATCH 01/61] big ugly initial commit rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- tests/test_callactivity.py | 2 +- tests/test_jump_to_wf.py | 2 +- tests/test_multi_user.py | 8 +- zengine/current.py | 2 +- zengine/lib/test_utils.py | 4 +- .../{notifications => messaging}/__init__.py | 10 +- zengine/messaging/model.py | 150 ++++++++++++++++++ zengine/models/__init__.py | 2 +- zengine/models/auth.py | 2 +- zengine/notifications/model.py | 49 ------ zengine/receivers.py | 2 +- zengine/settings.py | 3 +- zengine/views/auth.py | 2 +- 13 files changed, 170 insertions(+), 68 deletions(-) rename zengine/{notifications => messaging}/__init__.py (85%) create mode 100644 zengine/messaging/model.py delete mode 100644 zengine/notifications/model.py diff --git a/tests/test_callactivity.py b/tests/test_callactivity.py index 7d64983b..40f9f66b 100644 --- a/tests/test_callactivity.py +++ b/tests/test_callactivity.py @@ -12,7 +12,7 @@ from zengine.lib.exceptions import HTTPError from zengine.lib.test_utils import BaseTestCase from zengine.models import User -from zengine.notifications.model import NotificationMessage +from zengine.messaging.model import Message from zengine.signals import lane_user_change diff --git a/tests/test_jump_to_wf.py b/tests/test_jump_to_wf.py index 897c46e5..7e4a434b 100644 --- a/tests/test_jump_to_wf.py +++ b/tests/test_jump_to_wf.py @@ -12,7 +12,7 @@ from zengine.lib.exceptions import HTTPError from zengine.lib.test_utils import BaseTestCase from zengine.models import User -from zengine.notifications.model import NotificationMessage +from zengine.messaging.model import Message from zengine.signals import lane_user_change diff --git a/tests/test_multi_user.py b/tests/test_multi_user.py index 821fc820..f100b32e 100644 --- a/tests/test_multi_user.py +++ b/tests/test_multi_user.py @@ -12,7 +12,7 @@ from zengine.lib.exceptions import HTTPError from zengine.lib.test_utils import BaseTestCase from zengine.models import User -from zengine.notifications.model import NotificationMessage +from zengine.messaging.model import Message from zengine.signals import lane_user_change @@ -20,7 +20,7 @@ class TestCase(BaseTestCase): def test_multi_user_mono(self): test_user = User.objects.get(username='test_user') self.prepare_client('/multi_user2/', user=test_user) - with BlockSave(NotificationMessage): + with BlockSave(Message): resp = self.client.post() assert resp.json['msgbox']['title'] == settings.MESSAGES['lane_change_message_title'] token, user = self.get_user_token('test_user2') @@ -37,12 +37,12 @@ def mock(sender, *args, **kwargs): self.old_lane = kwargs['old_lane'] self.owner = list(kwargs['possible_owners'])[0] - NotificationMessage.objects.delete() + Message.objects.delete() lane_user_change.connect(mock) wf_name = '/multi_user/' self.prepare_client(wf_name, username='test_user') - with BlockSave(NotificationMessage): + with BlockSave(Message): self.client.post() token, user = self.get_user_token('test_user') assert self.owner.username == 'test_user' diff --git a/zengine/current.py b/zengine/current.py index 227cc592..efddca7c 100644 --- a/zengine/current.py +++ b/zengine/current.py @@ -24,7 +24,7 @@ from zengine.lib.cache import WFCache from zengine.lib.camunda_parser import CamundaBMPNParser from zengine.log import log -from zengine.notifications import Notify +from zengine.messaging import Notify DEFAULT_LANE_CHANGE_MSG = { 'title': settings.MESSAGES['lane_change_message_title'], diff --git a/zengine/lib/test_utils.py b/zengine/lib/test_utils.py index 703731f3..bcb24288 100644 --- a/zengine/lib/test_utils.py +++ b/zengine/lib/test_utils.py @@ -15,7 +15,7 @@ from zengine.wf_daemon import Worker from zengine.models import User -from zengine.notifications.model import NotificationMessage +from zengine.messaging.model import Message class ResponseWrapper(object): @@ -224,6 +224,6 @@ def _do_login(self): @staticmethod def get_user_token(username): user = User.objects.get(username=username) - msg = NotificationMessage.objects.filter(receiver=user)[0] + msg = Message.objects.filter(receiver=user)[0] token = msg.url.split('/')[-1] return token, user diff --git a/zengine/notifications/__init__.py b/zengine/messaging/__init__.py similarity index 85% rename from zengine/notifications/__init__.py rename to zengine/messaging/__init__.py index 0fca11cf..b76be91d 100644 --- a/zengine/notifications/__init__.py +++ b/zengine/messaging/__init__.py @@ -16,7 +16,7 @@ import time import six from zengine.lib.cache import Cache, KeepAlive -from .model import NotificationMessage +from .model import Message class Notify(Cache, ClientQueue): """ @@ -56,10 +56,10 @@ def _delayed_send(self, offline_messages): def set_message(self, title, msg, typ, url=None, sender=None): message = {'title': title, 'body': msg, 'type': typ, 'url': url, 'id': uuid4().hex} if sender and isinstance(sender, six.string_types): - sender = NotificationMessage.sender.objects.get(sender) - receiver = NotificationMessage.receiver.objects.get(self.user_id) - NotificationMessage(typ=typ, msg_title=title, body=msg, url=url, - sender=sender, receiver=receiver).save() + sender = Message.sender.objects.get(sender) + receiver = Message.receiver.objects.get(self.user_id) + Message(typ=typ, msg_title=title, body=msg, url=url, + sender=sender, receiver=receiver).save() if KeepAlive(user_id=self.user_id).is_alive(): client_message = {'cmd': 'notification', 'notifications': [message, ]} self.send_to_queue(client_message) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py new file mode 100644 index 00000000..3d7b5f5d --- /dev/null +++ b/zengine/messaging/model.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- +""" +""" + +# Copyright (C) 2015 ZetaOps Inc. +# +# This file is licensed under the GNU General Public License v3 +# (GPLv3). See LICENSE.txt for details. +import pika + +from pyoko import Model, field, ListNode +from pyoko.conf import settings +from pyoko.lib.utils import get_object_from_path +from zengine.client_queue import BLOCKING_MQ_PARAMS + +UserModel = get_object_from_path(settings.USER_MODEL) + +MSG_TYPES = ( + (1, "Info"), + (11, "Error"), + (111, "Success"), + (2, "Direct Message"), + (3, "Broadcast Message") + (4, "Channel Message") +) + +CHANNEL_TYPES = ( + (10, "System Broadcast"), + (10, "User Broadcast"), + (15, "Direct"), + (20, "Chat"), +) + + +MESSAGE_STATUS = ( + (1, "Created"), + (11, "Transmitted"), + (22, "Seen"), + (33, "Read"), + (44, "Archived"), + +) +ATTACHMENT_TYPES = ( + (1, "Document"), + (11, "Spreadsheet"), + (22, "Image"), + (33, "PDF"), + +) + + +def get_mq_connection(): + connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) + channel = connection.channel() + return connection, channel + +class Channel(Model): + name = field.String("Name") + code_name = field.String("Internal name") + description = field.String("Description") + owner = UserModel(reverse_name='created_channels') + typ = field.Integer("Type", choices=CHANNEL_TYPES) + + class Managers(ListNode): + user = UserModel(reverse_name='managed_channels') + + + def _connect_mq(self): + self.connection, self.channel = get_mq_connection() + return self.channel + + def create_exchange(self): + """ + This method creates MQ exch + which actually needed to be defined only once. + """ + channel = self._connect_mq() + channel.exchange_declare(exchange=self.code_name) + +class Subscription(Model): + """ + Permission model + """ + + channel = Channel() + user = UserModel(reverse_name='channels') + is_muted = field.Boolean("Mute the channel") + inform_me = field.Boolean("Inform when I'm mentioned") + can_leave = field.Boolean("Membership is not obligatory", default=True) + # status = field.Integer("Status", choices=SUBSCRIPTION_STATUS) + + def _connect_mq(self): + self.connection, self.channel = get_mq_connection() + return self.channel + + def create_exchange(self): + """ + This method creates user's private exchange + which actually needed to be defined only once. + """ + channel = self._connect_mq() + channel.exchange_declare(exchange=self.user.key) + + + + + def __unicode__(self): + return "%s in %s" % (self.user, self.channel.name) + +class Message(Model): + """ + Permission model + """ + + typ = field.Integer("Type", choices=MSG_TYPES) + status = field.Integer("Status", choices=MESSAGE_STATUS) + msg_title = field.String("Title") + body = field.String("Body") + url = field.String("URL") + channel = Channel() + sender = UserModel(reverse_name='sent_messages') + receiver = UserModel(reverse_name='received_messages') + + def __unicode__(self): + content = self.msg_title or self.body + return "%s%s" % (content[:30], '...' if len(content) > 30 else '') + + +class Attachment(Model): + """ + A model to store message attachments + """ + file = field.File("File", random_name=True, required=False) + typ = field.Integer("Type", choices=ATTACHMENT_TYPES) + name = field.String("Name") + description = field.String("Description") + channel = Channel() + message = Message() + + def __unicode__(self): + return self.name + + +class Favorite(Model): + """ + A model to store users favorited messages + """ + channel = Channel() + user = UserModel() + message = Message() diff --git a/zengine/models/__init__.py b/zengine/models/__init__.py index 9fd91777..6ac86ac8 100644 --- a/zengine/models/__init__.py +++ b/zengine/models/__init__.py @@ -7,4 +7,4 @@ # This file is licensed under the GNU General Public License v3 # (GPLv3). See LICENSE.txt for details. from .auth import * -from ..notifications.model import * +from ..messaging.model import * diff --git a/zengine/models/auth.py b/zengine/models/auth.py index 6c7d489b..7206a7a5 100644 --- a/zengine/models/auth.py +++ b/zengine/models/auth.py @@ -107,7 +107,7 @@ def get_role(self, role_id): return self.role_set.node_dict[role_id] def send_message(self, title, message, sender=None): - from zengine.notifications import Notify + from zengine.messaging import Notify Notify(self.key).set_message(title, message, typ=Notify.Message, sender=sender) diff --git a/zengine/notifications/model.py b/zengine/notifications/model.py deleted file mode 100644 index 83bb3337..00000000 --- a/zengine/notifications/model.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- -""" -""" - -# Copyright (C) 2015 ZetaOps Inc. -# -# This file is licensed under the GNU General Public License v3 -# (GPLv3). See LICENSE.txt for details. -from pyoko import Model, field, ListNode -from pyoko.conf import settings -from pyoko.lib.utils import get_object_from_path - -UserModel = get_object_from_path(settings.USER_MODEL) - -NOTIFY_MSG_TYPES = ( - (1, "Info"), - (11, "Error"), - (111, "Success"), - (2, "User Message"), - (3, "Broadcast Message") -) - - -NOTIFICATION_STATUS = ( - (1, "Created"), - (11, "Transmitted"), - (22, "Seen"), - (33, "Read"), - (44, "Archived"), - -) - -class NotificationMessage(Model): - """ - Permission model - """ - - typ = field.Integer("Message Type", choices=NOTIFY_MSG_TYPES) - status = field.Integer("Status", choices=NOTIFICATION_STATUS) - msg_title = field.String("Title") - body = field.String("Body") - url = field.String("URL") - sender = UserModel(reverse_name='sent_messages') - receiver = UserModel(reverse_name='received_messages') - - def __unicode__(self): - content = self.msg_title or self.body - return "%s%s" % (content[:30], '...' if len(content) > 30 else '') - diff --git a/zengine/receivers.py b/zengine/receivers.py index 965b2fdf..152e0394 100644 --- a/zengine/receivers.py +++ b/zengine/receivers.py @@ -37,7 +37,7 @@ def send_message_for_lane_change(sender, *args, **kwargs): from zengine.lib.catalog_data import gettxt as _ from pyoko.lib.utils import get_object_from_path UserModel = get_object_from_path(settings.USER_MODEL) - from zengine.notifications import Notify + from zengine.messaging import Notify current = kwargs['current'] old_lane = kwargs['old_lane'] owners = kwargs['possible_owners'] diff --git a/zengine/settings.py b/zengine/settings.py index 774d6158..5fc8bad2 100644 --- a/zengine/settings.py +++ b/zengine/settings.py @@ -111,10 +111,11 @@ #: View URL list for non-workflow views. #: -#: ('falcon URI template', 'python path to view method/class'), +#: ('URI template', 'python path to view method/class'), VIEW_URLS = { 'dashboard': 'zengine.views.menu.Menu', 'ping': 'zengine.views.dev_utils.Ping', + } if DEBUG: diff --git a/zengine/views/auth.py b/zengine/views/auth.py index 1c8f0b2c..66a340c0 100644 --- a/zengine/views/auth.py +++ b/zengine/views/auth.py @@ -10,7 +10,7 @@ from pyoko import fields from zengine.forms.json_form import JsonForm from zengine.lib.cache import UserSessionID, KeepAlive -from zengine.notifications import Notify +from zengine.messaging import Notify from zengine.views.base import SimpleView From 849249af224372270854902ed939bab8a1d7fe34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Tue, 21 Jun 2016 10:56:05 +0300 Subject: [PATCH 02/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/model.py | 109 +++++++++++++++++++++++-------------- zengine/views/auth.py | 13 +++++ 2 files changed, 82 insertions(+), 40 deletions(-) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 3d7b5f5d..e3182c34 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -6,6 +6,8 @@ # # This file is licensed under the GNU General Public License v3 # (GPLv3). See LICENSE.txt for details. +import json + import pika from pyoko import Model, field, ListNode @@ -15,46 +17,23 @@ UserModel = get_object_from_path(settings.USER_MODEL) -MSG_TYPES = ( - (1, "Info"), - (11, "Error"), - (111, "Success"), - (2, "Direct Message"), - (3, "Broadcast Message") - (4, "Channel Message") -) + +def get_mq_connection(): + connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) + channel = connection.channel() + return connection, channel + CHANNEL_TYPES = ( + # (1, "Notification"), (10, "System Broadcast"), - (10, "User Broadcast"), - (15, "Direct"), + (15, "User Broadcast"), (20, "Chat"), + (25, "Direct"), ) -MESSAGE_STATUS = ( - (1, "Created"), - (11, "Transmitted"), - (22, "Seen"), - (33, "Read"), - (44, "Archived"), - -) -ATTACHMENT_TYPES = ( - (1, "Document"), - (11, "Spreadsheet"), - (22, "Image"), - (33, "PDF"), - -) - - -def get_mq_connection(): - connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) - channel = connection.channel() - return connection, channel - -class Channel(Model): +class Channel(Model): name = field.String("Name") code_name = field.String("Internal name") description = field.String("Description") @@ -64,6 +43,11 @@ class Channel(Model): class Managers(ListNode): user = UserModel(reverse_name='managed_channels') + def add_message(self, body, title, sender=None, url=None, typ=2): + channel = self._connect_mq() + mq_msg = json.dumps(dict(sender=sender, body=body, msg_title=title, url=url, typ=typ)) + channel.basic_publish(exchange=self.code_name, body=mq_msg) + Message(sender=sender, body=body, msg_title=title, url=url, typ=typ, channel=self).save() def _connect_mq(self): self.connection, self.channel = get_mq_connection() @@ -71,11 +55,15 @@ def _connect_mq(self): def create_exchange(self): """ - This method creates MQ exch - which actually needed to be defined only once. + Creates MQ exchange for this channel + Needs to be defined only once. """ channel = self._connect_mq() - channel.exchange_declare(exchange=self.code_name) + channel.exchange_declare(exchange=self.code_name, exchange_type='fanout', durable=True) + + def post_creation(self): + self.create_exchange() + class Subscription(Model): """ @@ -87,6 +75,7 @@ class Subscription(Model): is_muted = field.Boolean("Mute the channel") inform_me = field.Boolean("Inform when I'm mentioned") can_leave = field.Boolean("Membership is not obligatory", default=True) + # status = field.Integer("Status", choices=SUBSCRIPTION_STATUS) def _connect_mq(self): @@ -95,18 +84,48 @@ def _connect_mq(self): def create_exchange(self): """ - This method creates user's private exchange - which actually needed to be defined only once. + Creates user's private exchange + Actually needed to be defined only once. + but since we don't know if it's exists or not + we always call it before """ channel = self._connect_mq() - channel.exchange_declare(exchange=self.user.key) - + channel.exchange_declare(exchange=self.user.key, exchange_type='direct', durable=True) + def bind_to_channel(self): + """ + Binds (subscribes) users private exchange to channel exchange + Automatically called at creation of subscription record. + """ + channel = self._connect_mq() + channel.exchange_bind(source=self.channel.code_name, destination=self.user.key) + def post_creation(self): + self.create_exchange() + self.bind_to_channel() def __unicode__(self): return "%s in %s" % (self.user, self.channel.name) + +MSG_TYPES = ( + (1, "Info"), + (11, "Error"), + (111, "Success"), + (2, "Direct Message"), + (3, "Broadcast Message") + (4, "Channel Message") +) +MESSAGE_STATUS = ( + (1, "Created"), + (11, "Transmitted"), + (22, "Seen"), + (33, "Read"), + (44, "Archived"), + +) + + class Message(Model): """ Permission model @@ -119,6 +138,7 @@ class Message(Model): url = field.String("URL") channel = Channel() sender = UserModel(reverse_name='sent_messages') + # FIXME: receiver should be removed after all of it's usages refactored to channels receiver = UserModel(reverse_name='received_messages') def __unicode__(self): @@ -126,6 +146,15 @@ def __unicode__(self): return "%s%s" % (content[:30], '...' if len(content) > 30 else '') +ATTACHMENT_TYPES = ( + (1, "Document"), + (11, "Spreadsheet"), + (22, "Image"), + (33, "PDF"), + +) + + class Attachment(Model): """ A model to store message attachments diff --git a/zengine/views/auth.py b/zengine/views/auth.py index 66a340c0..8afa7ce1 100644 --- a/zengine/views/auth.py +++ b/zengine/views/auth.py @@ -53,9 +53,21 @@ class Login(SimpleView): does the authentication at ``do`` stage. """ + def _do_binding(self): + """ + Bind user's ephemeral session queue to user's durable private exchange + """ + from zengine.messaging.model import get_mq_connection + connection, channel = get_mq_connection() + channel.queue_bind(exchange=self.current.user_id, + queue=self.current.session.sess_id, + # routing_key="#" + ) + def do_view(self): """ Authenticate user with given credentials. + Connects user's queue and exchange """ self.current.task_data['login_successful'] = False if self.current.is_auth: @@ -67,6 +79,7 @@ def do_view(self): self.current.input['password']) self.current.task_data['login_successful'] = auth_result if auth_result: + self._do_binding() user_sess = UserSessionID(self.current.user_id) old_sess_id = user_sess.get() user_sess.set(self.current.session.sess_id) From 989d1f9810cf6ec005e5f9766c7d0c74aaf71f99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Wed, 22 Jun 2016 10:27:57 +0300 Subject: [PATCH 03/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/management_commands.py | 17 ++++++++++++++ zengine/messaging/lib.py | 27 +++++++++++++++++++++ zengine/messaging/model.py | 31 +++++++++++++++---------- zengine/tornado_server/queue_manager.py | 14 ++++++----- zengine/wf_daemon.py | 4 ++-- 5 files changed, 73 insertions(+), 20 deletions(-) create mode 100644 zengine/messaging/lib.py diff --git a/zengine/management_commands.py b/zengine/management_commands.py index ee017898..1af0d76a 100644 --- a/zengine/management_commands.py +++ b/zengine/management_commands.py @@ -183,3 +183,20 @@ def run(self): else: worker = Worker() worker.run() + + +class PrepareMQ(Command): + """ + Creates necessary exchanges, queues and bindings + """ + CMD_NAME = 'preparemq' + HELP = 'Creates necessary exchanges, queues and bindings' + + def run(self): + from zengine.wf_daemon import run_workers, Worker + worker_count = int(self.manager.args.workers or 1) + if worker_count > 1: + run_workers(worker_count) + else: + worker = Worker() + worker.run() diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py new file mode 100644 index 00000000..a4de5db1 --- /dev/null +++ b/zengine/messaging/lib.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +""" +""" + +# Copyright (C) 2015 ZetaOps Inc. +# +# This file is licensed under the GNU General Public License v3 +# (GPLv3). See LICENSE.txt for details. +import pika + + +class BaseUser(object): + connection = None + channel = None + + + def _connect_mq(self): + if not self.connection is None or self.connection.is_closed: + self.connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) + self.channel = selfconnection.channel() + return self.channel + + + def send_message(self, title, message, sender=None, url=None, typ=1): + channel = self._connect_mq() + mq_msg = json.dumps(dict(sender=sender, body=message, msg_title=title, url=url, typ=typ)) + channel.basic_publish(exchange=self.key, body=mq_msg) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index e3182c34..b3360528 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -18,27 +18,34 @@ UserModel = get_object_from_path(settings.USER_MODEL) + def get_mq_connection(): connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) channel = connection.channel() return connection, channel -CHANNEL_TYPES = ( - # (1, "Notification"), - (10, "System Broadcast"), - (15, "User Broadcast"), - (20, "Chat"), - (25, "Direct"), -) +# CHANNEL_TYPES = ( +# (1, "Notification"), + # (10, "System Broadcast"), + # (20, "Chat"), + # (25, "Direct"), +# ) class Channel(Model): + channel = None + connection = None + name = field.String("Name") code_name = field.String("Internal name") description = field.String("Description") owner = UserModel(reverse_name='created_channels') - typ = field.Integer("Type", choices=CHANNEL_TYPES) + # is this users private exchange + is_private = field.Boolean() + # is this a One-To-One channel + is_direct = field.Boolean() + # typ = field.Integer("Type", choices=CHANNEL_TYPES) class Managers(ListNode): user = UserModel(reverse_name='managed_channels') @@ -50,7 +57,8 @@ def add_message(self, body, title, sender=None, url=None, typ=2): Message(sender=sender, body=body, msg_title=title, url=url, typ=typ, channel=self).save() def _connect_mq(self): - self.connection, self.channel = get_mq_connection() + if not self.connection is None or self.connection.is_closed: + self.connection, self.channel = get_mq_connection() return self.channel def create_exchange(self): @@ -113,7 +121,7 @@ def __unicode__(self): (11, "Error"), (111, "Success"), (2, "Direct Message"), - (3, "Broadcast Message") + (3, "Broadcast Message"), (4, "Channel Message") ) MESSAGE_STATUS = ( @@ -130,7 +138,6 @@ class Message(Model): """ Permission model """ - typ = field.Integer("Type", choices=MSG_TYPES) status = field.Integer("Status", choices=MESSAGE_STATUS) msg_title = field.String("Title") @@ -172,7 +179,7 @@ def __unicode__(self): class Favorite(Model): """ - A model to store users favorited messages + A model to store users bookmarked messages """ channel = Channel() user = UserModel() diff --git a/zengine/tornado_server/queue_manager.py b/zengine/tornado_server/queue_manager.py index 982e3fce..a35427d3 100644 --- a/zengine/tornado_server/queue_manager.py +++ b/zengine/tornado_server/queue_manager.py @@ -66,7 +66,7 @@ def create_channel(self): def _send_message(self, sess_id, input_data): log.info("sending data for %s" % sess_id) - self.input_channel.basic_publish(exchange='tornado_input', + self.input_channel.basic_publish(exchange='input_exc', routing_key=sess_id, body=json_encode(input_data)) @@ -160,7 +160,7 @@ def on_conn_open(self, channel): Args: channel: input channel """ - self.in_channel.exchange_declare(exchange='tornado_input', type='topic') + self.in_channel.exchange_declare(exchange='input_exc', type='topic', durable=True) channel.queue_declare(callback=self.on_input_queue_declare, queue=self.INPUT_QUEUE_NAME) def on_input_queue_declare(self, queue): @@ -172,7 +172,7 @@ def on_input_queue_declare(self, queue): queue: input queue """ self.in_channel.queue_bind(callback=None, - exchange='tornado_input', + exchange='input_exc', queue=self.INPUT_QUEUE_NAME, routing_key="#") @@ -212,10 +212,12 @@ def _on_output_queue_decleration(queue): self.connection.channel(_on_output_channel_creation) def redirect_incoming_message(self, sess_id, message, request): - message = message[:-1] + ',"_zops_remote_ip":"%s"}' % request.remote_ip - self.in_channel.basic_publish(exchange='tornado_input', + message = json_decode(message) + message['_zops_sess_id'] = sess_id + message['_zops_remote_ip'] = request.remote_ip + self.in_channel.basic_publish(exchange='input_exc', routing_key=sess_id, - body=message) + body=json_encode(message)) def on_message(self, channel, method, header, body): sess_id = method.routing_key diff --git a/zengine/wf_daemon.py b/zengine/wf_daemon.py index 7f320eab..4002bda3 100755 --- a/zengine/wf_daemon.py +++ b/zengine/wf_daemon.py @@ -36,7 +36,7 @@ class Worker(object): Workflow runner worker object """ INPUT_QUEUE_NAME = 'in_queue' - INPUT_EXCHANGE = 'tornado_input' + INPUT_EXCHANGE = 'input_exc' def __init__(self): self.connect() @@ -62,7 +62,7 @@ def connect(self): self.client_queue = ClientQueue() self.input_channel = self.connection.channel() - self.input_channel.exchange_declare(exchange=self.INPUT_EXCHANGE, type='topic') + self.input_channel.exchange_declare(exchange=self.INPUT_EXCHANGE, type='topic', durable=True) self.input_channel.queue_declare(queue=self.INPUT_QUEUE_NAME) self.input_channel.queue_bind(exchange=self.INPUT_EXCHANGE, queue=self.INPUT_QUEUE_NAME) log.info("Bind to queue named '%s' queue with exchange '%s'" % (self.INPUT_QUEUE_NAME, self.INPUT_EXCHANGE)) From bfdfc1089fbdb827c67a812d6f330af407ad7860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Wed, 22 Jun 2016 18:22:33 +0300 Subject: [PATCH 04/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/management_commands.py | 24 ++++++--- zengine/messaging/__init__.py | 3 +- zengine/messaging/lib.py | 95 ++++++++++++++++++++++++++++++++-- zengine/messaging/model.py | 47 ++++++++++------- zengine/models/auth.py | 49 ++---------------- 5 files changed, 140 insertions(+), 78 deletions(-) diff --git a/zengine/management_commands.py b/zengine/management_commands.py index 1af0d76a..57f923fb 100644 --- a/zengine/management_commands.py +++ b/zengine/management_commands.py @@ -9,6 +9,7 @@ import six from pyoko.exceptions import ObjectDoesNotExist +from pyoko.lib.utils import get_object_from_path from pyoko.manage import * from zengine.views.crud import SelectBoxCache @@ -193,10 +194,19 @@ class PrepareMQ(Command): HELP = 'Creates necessary exchanges, queues and bindings' def run(self): - from zengine.wf_daemon import run_workers, Worker - worker_count = int(self.manager.args.workers or 1) - if worker_count > 1: - run_workers(worker_count) - else: - worker = Worker() - worker.run() + self.create_user_channels() + self.create_exchanges() + + def create_user_channels(self): + from zengine.messaging.model import Channel + user_model = get_object_from_path(settings.USER_MODEL) + for usr in user_model.objects.filter(): + ch, new = Channel.objects.get_or_create(owner=usr, is_private=True) + print("%s exchange: %s" % ('created' if new else 'existing', ch.name)) + + def create_channel_exchanges(self): + from zengine.messaging.model import Channel + for ch in Channel.objects.filter(): + print("(re)creation exchange: %s" % ch.name) + ch.create_exchange() + diff --git a/zengine/messaging/__init__.py b/zengine/messaging/__init__.py index b76be91d..588e4ac4 100644 --- a/zengine/messaging/__init__.py +++ b/zengine/messaging/__init__.py @@ -16,7 +16,7 @@ import time import six from zengine.lib.cache import Cache, KeepAlive -from .model import Message + class Notify(Cache, ClientQueue): """ @@ -54,6 +54,7 @@ def _delayed_send(self, offline_messages): self.remove_item(n) def set_message(self, title, msg, typ, url=None, sender=None): + from .model import Message message = {'title': title, 'body': msg, 'type': typ, 'url': url, 'id': uuid4().hex} if sender and isinstance(sender, six.string_types): sender = Message.sender.objects.get(sender) diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py index a4de5db1..3d85313d 100644 --- a/zengine/messaging/lib.py +++ b/zengine/messaging/lib.py @@ -6,22 +6,107 @@ # # This file is licensed under the GNU General Public License v3 # (GPLv3). See LICENSE.txt for details. +import json + import pika +from passlib.handlers.pbkdf2 import pbkdf2_sha512 + +from pyoko.conf import settings +from zengine.client_queue import BLOCKING_MQ_PARAMS + class BaseUser(object): connection = None channel = None - def _connect_mq(self): if not self.connection is None or self.connection.is_closed: self.connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) - self.channel = selfconnection.channel() + self.channel = self.connection.channel() return self.channel + def get_avatar_url(self): + """ + Bu metot kullanıcıya ait avatar url'ini üretir. + + Returns: + str: kullanıcı avatar url + """ + return "%s%s" % (settings.S3_PUBLIC_URL, self.avatar) + + def __unicode__(self): + return "User %s" % self.username + + def set_password(self, raw_password): + """ + Kullanıcı şifresini encrypt ederek set eder. + + Args: + raw_password (str) + """ + self.password = pbkdf2_sha512.encrypt(raw_password, rounds=10000, + salt_size=10) + + def pre_save(self): + """ encrypt password if not already encrypted """ + if self.password and not self.password.startswith('$pbkdf2'): + self.set_password(self.password) + + def check_password(self, raw_password): + """ + Verilen encrypt edilmemiş şifreyle kullanıcıya ait encrypt + edilmiş şifreyi karşılaştırır. + + Args: + raw_password (str) + + Returns: + bool: Değerler aynı olması halinde True, değilse False + döner. + """ + return pbkdf2_sha512.verify(raw_password, self.password) + + def get_role(self, role_id): + """ + Kullanıcıya ait Role nesnesini getirir. + + Args: + role_id (int) + + Returns: + dict: Role nesnesi + + """ + return self.role_set.node_dict[role_id] + + @property + def full_name(self): + return self.username def send_message(self, title, message, sender=None, url=None, typ=1): - channel = self._connect_mq() - mq_msg = json.dumps(dict(sender=sender, body=message, msg_title=title, url=url, typ=typ)) - channel.basic_publish(exchange=self.key, body=mq_msg) + """ + sends message to users private mq exchange + Args: + title: + message: + sender: + url: + typ: + + + """ + mq_channel = self._connect_mq() + mq_msg = dict(body=message, msg_title=title, url=url, typ=typ) + if sender: + mq_msg['sender_name'] = sender.full_name + mq_msg['sender_key'] = sender.key + + mq_channel.basic_publish(exchange=self.key, body=json.dumps(mq_msg)) + self._write_message(sender, message, title, url, typ) + + def _write_message(self, sender, body, title, url, typ): + from zengine.messaging.model import Channel, Message + channel = Channel.objects.get(owner=self, is_private=True) + Message(channel=channel, sender=sender, msg_title=title, + body=body, receiver=self, url=url, typ=typ).save() diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index b3360528..dfa6431e 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -18,7 +18,6 @@ UserModel = get_object_from_path(settings.USER_MODEL) - def get_mq_connection(): connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) channel = connection.channel() @@ -27,9 +26,9 @@ def get_mq_connection(): # CHANNEL_TYPES = ( # (1, "Notification"), - # (10, "System Broadcast"), - # (20, "Chat"), - # (25, "Direct"), +# (10, "System Broadcast"), +# (20, "Chat"), +# (25, "Direct"), # ) @@ -45,8 +44,12 @@ class Channel(Model): is_private = field.Boolean() # is this a One-To-One channel is_direct = field.Boolean() + # typ = field.Integer("Type", choices=CHANNEL_TYPES) + class Meta: + unique_together = (('is_private', 'owner'),) + class Managers(ListNode): user = UserModel(reverse_name='managed_channels') @@ -56,10 +59,11 @@ def add_message(self, body, title, sender=None, url=None, typ=2): channel.basic_publish(exchange=self.code_name, body=mq_msg) Message(sender=sender, body=body, msg_title=title, url=url, typ=typ, channel=self).save() - def _connect_mq(self): - if not self.connection is None or self.connection.is_closed: - self.connection, self.channel = get_mq_connection() - return self.channel + @classmethod + def _connect_mq(cls): + if cls.connection is None or cls.connection.is_closed: + cls.connection, cls.channel = get_mq_connection() + return cls.channel def create_exchange(self): """ @@ -69,6 +73,10 @@ def create_exchange(self): channel = self._connect_mq() channel.exchange_declare(exchange=self.code_name, exchange_type='fanout', durable=True) + def pre_creation(self): + if not self.code_name: + self.code_name = self.key + def post_creation(self): self.create_exchange() @@ -86,16 +94,18 @@ class Subscription(Model): # status = field.Integer("Status", choices=SUBSCRIPTION_STATUS) - def _connect_mq(self): - self.connection, self.channel = get_mq_connection() - return self.channel + @classmethod + def _connect_mq(cls): + if cls.connection is None or cls.connection.is_closed: + cls.connection, cls.channel = get_mq_connection() + return cls.channel def create_exchange(self): """ Creates user's private exchange Actually needed to be defined only once. but since we don't know if it's exists or not - we always call it before + we always call it before binding it to related channel """ channel = self._connect_mq() channel.exchange_declare(exchange=self.user.key, exchange_type='direct', durable=True) @@ -117,9 +127,9 @@ def __unicode__(self): MSG_TYPES = ( - (1, "Info"), - (11, "Error"), - (111, "Success"), + (1, "Info Notification"), + (11, "Error Notification"), + (111, "Success Notification"), (2, "Direct Message"), (3, "Broadcast Message"), (4, "Channel Message") @@ -138,15 +148,14 @@ class Message(Model): """ Permission model """ + channel = Channel() + sender = UserModel(reverse_name='sent_messages') + receiver = UserModel(reverse_name='received_messages') typ = field.Integer("Type", choices=MSG_TYPES) status = field.Integer("Status", choices=MESSAGE_STATUS) msg_title = field.String("Title") body = field.String("Body") url = field.String("URL") - channel = Channel() - sender = UserModel(reverse_name='sent_messages') - # FIXME: receiver should be removed after all of it's usages refactored to channels - receiver = UserModel(reverse_name='received_messages') def __unicode__(self): content = self.msg_title or self.body diff --git a/zengine/models/auth.py b/zengine/models/auth.py index 7206a7a5..deb3af65 100644 --- a/zengine/models/auth.py +++ b/zengine/models/auth.py @@ -9,6 +9,7 @@ from pyoko import Model, field, ListNode from passlib.hash import pbkdf2_sha512 +from zengine.messaging.lib import BaseUser class Permission(Model): @@ -41,48 +42,20 @@ def get_permitted_roles(self): return [rset.role for rset in self.role_set] -class User(Model): +class User(Model, BaseUser): """ Basic User model """ username = field.String("Username", index=True) password = field.String("Password") superuser = field.Boolean("Super user", default=False) + avatar = field.File("Avatar", random_name=True, required=False) class Meta: """ meta class """ list_fields = ['username', 'superuser'] - def __unicode__(self): - return "User %s" % self.username - - def __repr__(self): - return "User_%s" % self.key - - def set_password(self, raw_password): - """ - Encrypts user password. - - Args: - raw_password: Clean password string. - - """ - self.password = pbkdf2_sha512.encrypt(raw_password, - rounds=10000, - salt_size=10) - - def check_password(self, raw_password): - """ - Checks given clean password against stored encrtyped password. - - Args: - raw_password: Clean password. - - Returns: - Boolean. True if given password match. - """ - return pbkdf2_sha512.verify(raw_password, self.password) def get_permissions(self): """ @@ -94,22 +67,6 @@ def get_permissions(self): users_primary_role = self.role_set[0].role return users_primary_role.get_permissions() - def get_role(self, role_id): - """ - Gets the first role of the user with given key. - - Args: - role_id: Key of the Role object. - - Returns: - :class:`Role` object - """ - return self.role_set.node_dict[role_id] - - def send_message(self, title, message, sender=None): - from zengine.messaging import Notify - Notify(self.key).set_message(title, message, typ=Notify.Message, sender=sender) - class Role(Model): """ From 6ebcf44a74cf079cacb5c2fc2bed5f2a8b81badd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Thu, 23 Jun 2016 14:53:28 +0300 Subject: [PATCH 05/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/lib/utils.py | 17 ++++++++++- zengine/management_commands.py | 6 ++-- zengine/messaging/lib.py | 4 +-- zengine/messaging/model.py | 54 ++++++++++++++++++++++++++++++---- zengine/messaging/views.py | 48 ++++++++++++++++++++++++++++++ 5 files changed, 118 insertions(+), 11 deletions(-) create mode 100644 zengine/messaging/views.py diff --git a/zengine/lib/utils.py b/zengine/lib/utils.py index fd971131..7acd46bb 100644 --- a/zengine/lib/utils.py +++ b/zengine/lib/utils.py @@ -1,4 +1,4 @@ - +# -*- coding: utf-8 -*- class DotDict(dict): """ @@ -6,6 +6,7 @@ class DotDict(dict): Slower that pure dict. """ + def __getattr__(self, attr): return self.get(attr, None) @@ -17,10 +18,24 @@ def date_to_solr(d): """ converts DD-MM-YYYY to YYYY-MM-DDT00:00:00Z""" return "{y}-{m}-{day}T00:00:00Z".format(day=d[:2], m=d[3:5], y=d[6:]) if d else d + def solr_to_date(d): """ converts YYYY-MM-DDT00:00:00Z to DD-MM-YYYY """ return "{day}:{m}:{y}".format(y=d[:4], m=d[5:7], day=d[8:10]) if d else d + def solr_to_year(d): """ converts YYYY-MM-DDT00:00:00Z to DD-MM-YYYY """ return d[:4] + +import re +def to_safe_str(s): + """ + converts some (tr) non-ascii chars to ascii counterparts, + then return the result as lowercase + """ + # TODO: This is insufficient as it doesn't do anything for other non-ascii chars + return re.sub(r'[^0-9a-zA-Z]+', '_', s.strip().replace(u'ğ', 'g').replace(u'ö', 'o').replace( + u'ç', 'c').replace(u'Ç','c').replace(u'Ö', u'O').replace(u'Ş', 's').replace( + u'Ü', 'u').replace(u'ı', 'i').replace(u'İ','i').replace(u'Ğ', 'g').replace( + u'ö', 'o').replace(u'ş', 's').replace(u'ü', 'u').lower(), re.UNICODE) diff --git a/zengine/management_commands.py b/zengine/management_commands.py index 57f923fb..78acd24b 100644 --- a/zengine/management_commands.py +++ b/zengine/management_commands.py @@ -195,18 +195,18 @@ class PrepareMQ(Command): def run(self): self.create_user_channels() - self.create_exchanges() + self.create_channel_exchanges() def create_user_channels(self): from zengine.messaging.model import Channel user_model = get_object_from_path(settings.USER_MODEL) for usr in user_model.objects.filter(): ch, new = Channel.objects.get_or_create(owner=usr, is_private=True) - print("%s exchange: %s" % ('created' if new else 'existing', ch.name)) + print("%s exchange: %s" % ('created' if new else 'existing', ch.code_name)) def create_channel_exchanges(self): from zengine.messaging.model import Channel for ch in Channel.objects.filter(): - print("(re)creation exchange: %s" % ch.name) + print("(re)creation exchange: %s" % ch.code_name) ch.create_exchange() diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py index 3d85313d..ac62a893 100644 --- a/zengine/messaging/lib.py +++ b/zengine/messaging/lib.py @@ -103,9 +103,9 @@ def send_message(self, title, message, sender=None, url=None, typ=1): mq_msg['sender_key'] = sender.key mq_channel.basic_publish(exchange=self.key, body=json.dumps(mq_msg)) - self._write_message(sender, message, title, url, typ) + self._write_message_to_db(sender, message, title, url, typ) - def _write_message(self, sender, body, title, url, typ): + def _write_message_to_db(self, sender, body, title, url, typ): from zengine.messaging.model import Channel, Message channel = Channel.objects.get(owner=self, is_private=True) Message(channel=channel, sender=sender, msg_title=title, diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index dfa6431e..899ab507 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -12,8 +12,10 @@ from pyoko import Model, field, ListNode from pyoko.conf import settings +from pyoko.exceptions import IntegrityError from pyoko.lib.utils import get_object_from_path from zengine.client_queue import BLOCKING_MQ_PARAMS +from zengine.lib.utils import to_safe_str UserModel = get_object_from_path(settings.USER_MODEL) @@ -33,6 +35,15 @@ def get_mq_connection(): class Channel(Model): + """ + Represents MQ exchanges. + + is_private: Represents users exchange hub + Each user have a durable private exchange, + which their code_name composed from user key prefixed with "prv_" + + is_direct: Represents a user-to-user direct message exchange + """ channel = None connection = None @@ -53,11 +64,37 @@ class Meta: class Managers(ListNode): user = UserModel(reverse_name='managed_channels') - def add_message(self, body, title, sender=None, url=None, typ=2): + @classmethod + def get_or_create_direct_channel(cls, initiator, receiver): + """ + Creates a direct messaging channel between two user + + Args: + initiator: User, who sent the first message + receiver: User, other party + + Returns: + Channel + """ + existing = cls.objects.or_filter( + code_name='%s_%s' % (initiator.key, receiver.key)).or_filter( + code_name='%s_%s' % (receiver.key, initiator.key)) + if existing: + return existing[0] + else: + channel_name = '%s_%s' % (initiator.key, receiver.key) + channel = cls(is_direct=True, code_name=channel_name).save() + Subscription(channel=channel, user=initiator).save() + Subscription(channel=channel, user=receiver).save() + return channel + + + def add_message(self, body, title, sender=None, url=None, typ=2, receiver=None): channel = self._connect_mq() mq_msg = json.dumps(dict(sender=sender, body=body, msg_title=title, url=url, typ=typ)) channel.basic_publish(exchange=self.code_name, body=mq_msg) - Message(sender=sender, body=body, msg_title=title, url=url, typ=typ, channel=self).save() + Message(sender=sender, body=body, msg_title=title, url=url, + typ=typ, channel=self, receiver=receiver).save() @classmethod def _connect_mq(cls): @@ -75,7 +112,13 @@ def create_exchange(self): def pre_creation(self): if not self.code_name: - self.code_name = self.key + if self.name: + self.code_name = to_safe_str(self.name) + return + if self.owner and self.is_private: + self.code_name = "prv_%s" % to_safe_str(self.owner.key) + return + raise IntegrityError('Non-private and non-direct channels should have a "name".') def post_creation(self): self.create_exchange() @@ -89,7 +132,8 @@ class Subscription(Model): channel = Channel() user = UserModel(reverse_name='channels') is_muted = field.Boolean("Mute the channel") - inform_me = field.Boolean("Inform when I'm mentioned") + inform_me = field.Boolean("Inform when I'm mentioned", default=True) + visible = field.Boolean("Show under user's channel list", default=True) can_leave = field.Boolean("Membership is not obligatory", default=True) # status = field.Integer("Status", choices=SUBSCRIPTION_STATUS) @@ -108,7 +152,7 @@ def create_exchange(self): we always call it before binding it to related channel """ channel = self._connect_mq() - channel.exchange_declare(exchange=self.user.key, exchange_type='direct', durable=True) + channel.exchange_declare(exchange=self.user.key, exchange_type='fanout', durable=True) def bind_to_channel(self): """ diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py new file mode 100644 index 00000000..65ef89bf --- /dev/null +++ b/zengine/messaging/views.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +""" +""" + +# Copyright (C) 2015 ZetaOps Inc. +# +# This file is licensed under the GNU General Public License v3 +# (GPLv3). See LICENSE.txt for details. +from pyoko.conf import settings +from pyoko.lib.utils import get_object_from_path +from zengine.views.base import BaseView +UserModel = get_object_from_path(settings.USER_MODEL) + +class MessageView(BaseView): + + def create_message(self): + """ + Creates a message for the given channel. + + Args: + self.current.input['data']['message'] = { + 'channel': code_name of the channel + 'title': Title of the message, optional + 'body': Title of the message + 'attachment':{ + 'name': title/name of file + 'key': storage key + } + } + + """ + # TODO: Attachment support!!! + msg = self.current.input['message'] + + # UserModel.objects.get(msg['receiver']).send_message(msg.get('title'), msg['body'], typ=2, + # sender=self.current.user) + + + + def new_broadcast_message(self): + pass + + def show_channel(self): + pass + + + def list_channels(self): + pass From 7ec60036827e1cd45fe1cff15214cdee85a9d792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Sun, 26 Jun 2016 03:54:44 +0300 Subject: [PATCH 06/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/model.py | 17 ++++++------ zengine/messaging/views.py | 57 ++++++++++++++++++++++++++++---------- 2 files changed, 50 insertions(+), 24 deletions(-) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 899ab507..aa6b9e06 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -76,8 +76,8 @@ def get_or_create_direct_channel(cls, initiator, receiver): Returns: Channel """ - existing = cls.objects.or_filter( - code_name='%s_%s' % (initiator.key, receiver.key)).or_filter( + existing = cls.objects.OR().filter( + code_name='%s_%s' % (initiator.key, receiver.key)).filter( code_name='%s_%s' % (receiver.key, initiator.key)) if existing: return existing[0] @@ -88,13 +88,12 @@ def get_or_create_direct_channel(cls, initiator, receiver): Subscription(channel=channel, user=receiver).save() return channel - - def add_message(self, body, title, sender=None, url=None, typ=2, receiver=None): - channel = self._connect_mq() + def add_message(self, body, title=None, sender=None, url=None, typ=2, receiver=None): + mq_channel = self._connect_mq() mq_msg = json.dumps(dict(sender=sender, body=body, msg_title=title, url=url, typ=typ)) - channel.basic_publish(exchange=self.code_name, body=mq_msg) - Message(sender=sender, body=body, msg_title=title, url=url, - typ=typ, channel=self, receiver=receiver).save() + mq_channel.basic_publish(exchange=self.code_name, body=mq_msg) + return Message(sender=sender, body=body, msg_title=title, url=url, + typ=typ, channel=self, receiver=receiver).save() @classmethod def _connect_mq(cls): @@ -221,7 +220,7 @@ class Attachment(Model): """ file = field.File("File", random_name=True, required=False) typ = field.Integer("Type", choices=ATTACHMENT_TYPES) - name = field.String("Name") + name = field.String("File Name") description = field.String("Description") channel = Channel() message = Message() diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 65ef89bf..779eb195 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -8,41 +8,68 @@ # (GPLv3). See LICENSE.txt for details. from pyoko.conf import settings from pyoko.lib.utils import get_object_from_path +from zengine.messaging.model import Channel, Attachment from zengine.views.base import BaseView + UserModel = get_object_from_path(settings.USER_MODEL) -class MessageView(BaseView): +class MessageView(BaseView): def create_message(self): """ Creates a message for the given channel. - Args: + API: self.current.input['data']['message'] = { - 'channel': code_name of the channel - 'title': Title of the message, optional - 'body': Title of the message - 'attachment':{ - 'name': title/name of file - 'key': storage key - } + 'channel': code_name of the channel. + 'receiver': Key of receiver. Can be blank for non-direct messages. + 'title': Title of the message. Can be blank. + 'body': Message body. + 'type': zengine.messaging.model.MSG_TYPES + 'attachments': [{ + 'description': Can be blank. + 'name': File name with extension. + 'content': base64 encoded file content + }] } """ - # TODO: Attachment support!!! msg = self.current.input['message'] + ch = Channel.objects.get(msg['channel']) + msg_obj = ch.add_message(body=msg['body'], typ=msg['typ'], sender=self.current.user, + title=msg['title'], receiver=msg['receiver'] or None) + if 'attachment' in msg: + for atch in msg['attachments']: + # TODO: Attachment type detection + typ = self._dedect_file_type(atch['name'], atch['content']) + Attachment(channel=ch, msg=msg_obj, name=atch['name'], file=atch['content'], + description=atch['description'], typ=typ).save() - # UserModel.objects.get(msg['receiver']).send_message(msg.get('title'), msg['body'], typ=2, - # sender=self.current.user) + def _dedect_file_type(self, name, content): + # TODO: Attachment type detection + return 1 # Document + def show_channel(self): + pass + def list_channels(self): + pass - def new_broadcast_message(self): + def create_public_channel(self): pass - def show_channel(self): + def create_direct_channel(self): + """ + Create a One-To-One channel for current user and selected user. + + """ pass + def find_message(self): + pass - def list_channels(self): + def delete_message(self): + pass + + def edit_message(self): pass From be402a4b03509e0523172e6e2bb86db8ae0f2166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Mon, 27 Jun 2016 13:59:32 +0300 Subject: [PATCH 07/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- docs/make.bat | 263 +++++++++++++++++++++++++++++++++++++ docs/zengine.messaging.rst | 36 +++++ docs/zengine.models.rst | 22 ++++ docs/zengine.rst | 10 ++ zengine/messaging/views.py | 126 ++++++++++-------- zengine/settings.py | 1 + 6 files changed, 400 insertions(+), 58 deletions(-) create mode 100644 docs/make.bat create mode 100644 docs/zengine.messaging.rst create mode 100644 docs/zengine.models.rst diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..d6b86778 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,263 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + echo. coverage to run coverage check of the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +REM Check if sphinx-build is available and fallback to Python version if any +%SPHINXBUILD% 1>NUL 2>NUL +if errorlevel 9009 goto sphinx_python +goto sphinx_ok + +:sphinx_python + +set SPHINXBUILD=python -m sphinx.__init__ +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +:sphinx_ok + + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\zengine.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\zengine.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "coverage" ( + %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage + if errorlevel 1 exit /b 1 + echo. + echo.Testing of coverage in the sources finished, look at the ^ +results in %BUILDDIR%/coverage/python.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end diff --git a/docs/zengine.messaging.rst b/docs/zengine.messaging.rst new file mode 100644 index 00000000..8889eed0 --- /dev/null +++ b/docs/zengine.messaging.rst @@ -0,0 +1,36 @@ +zengine.messaging package +========================= + + +zengine.messaging.lib module +---------------------------- + +.. automodule:: zengine.messaging.lib + :members: + :undoc-members: + :show-inheritance: + +zengine.messaging.model module +------------------------------ + +.. automodule:: zengine.messaging.model + :members: + :undoc-members: + :show-inheritance: + +zengine.messaging.views module +------------------------------ + +.. automodule:: zengine.messaging.views + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: zengine.messaging + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/zengine.models.rst b/docs/zengine.models.rst new file mode 100644 index 00000000..16553fe6 --- /dev/null +++ b/docs/zengine.models.rst @@ -0,0 +1,22 @@ +zengine.models package +====================== + +Submodules +---------- + +zengine.models.auth module +-------------------------- + +.. automodule:: zengine.models.auth + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: zengine.models + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/zengine.rst b/docs/zengine.rst index 92a03a82..ef5616c2 100644 --- a/docs/zengine.rst +++ b/docs/zengine.rst @@ -14,6 +14,8 @@ zengine package zengine.dispatch zengine.forms zengine.lib + zengine.messaging + zengine.models zengine.tornado_server zengine.views @@ -37,6 +39,14 @@ zengine.engine module :undoc-members: :show-inheritance: +zengine.messaging module +--------------------- + +.. automodule:: zengine.messaging + :members: + :undoc-members: + :show-inheritance: + zengine.log module ------------------ diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 779eb195..21e8b28e 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -14,62 +14,72 @@ UserModel = get_object_from_path(settings.USER_MODEL) -class MessageView(BaseView): - def create_message(self): - """ - Creates a message for the given channel. - - API: - self.current.input['data']['message'] = { - 'channel': code_name of the channel. - 'receiver': Key of receiver. Can be blank for non-direct messages. - 'title': Title of the message. Can be blank. - 'body': Message body. - 'type': zengine.messaging.model.MSG_TYPES + +def create_message(current): + """ + Creates a message for the given channel. + + API: + + .. code-block:: python + + {'view':'_zops_create_message', + 'message': { + 'channel': "code_name of the channel", + 'receiver': "Key of receiver. Can be blank for non-direct messages", + 'title': "Title of the message. Can be blank.", + 'body': "Message body.", + 'type': zengine.messaging.model.MSG_TYPES, 'attachments': [{ - 'description': Can be blank. - 'name': File name with extension. - 'content': base64 encoded file content - }] - } - - """ - msg = self.current.input['message'] - ch = Channel.objects.get(msg['channel']) - msg_obj = ch.add_message(body=msg['body'], typ=msg['typ'], sender=self.current.user, - title=msg['title'], receiver=msg['receiver'] or None) - if 'attachment' in msg: - for atch in msg['attachments']: - # TODO: Attachment type detection - typ = self._dedect_file_type(atch['name'], atch['content']) - Attachment(channel=ch, msg=msg_obj, name=atch['name'], file=atch['content'], - description=atch['description'], typ=typ).save() - - def _dedect_file_type(self, name, content): - # TODO: Attachment type detection - return 1 # Document - - def show_channel(self): - pass - - def list_channels(self): - pass - - def create_public_channel(self): - pass - - def create_direct_channel(self): - """ - Create a One-To-One channel for current user and selected user. - - """ - pass - - def find_message(self): - pass - - def delete_message(self): - pass - - def edit_message(self): - pass + 'description': "Can be blank.", + 'name': "File name with extension.", + 'content': "base64 encoded file content" + }]} + + """ + msg = current.input['message'] + ch = Channel.objects.get(msg['channel']) + msg_obj = ch.add_message(body=msg['body'], typ=msg['typ'], sender=current.user, + title=msg['title'], receiver=msg['receiver'] or None) + if 'attachment' in msg: + for atch in msg['attachments']: + # TODO: Attachment type detection + typ = current._dedect_file_type(atch['name'], atch['content']) + Attachment(channel=ch, msg=msg_obj, name=atch['name'], file=atch['content'], + description=atch['description'], typ=typ).save() + +def _dedect_file_type(current, name, content): + # TODO: Attachment type detection + return 1 # Document + +def show_channel(current): + """ + Initial display of channel content + + API: + + + """ + + +def list_channels(current): + pass + +def create_public_channel(current): + pass + +def create_direct_channel(current): + """ + Create a One-To-One channel for current user and selected user. + + """ + pass + +def find_message(current): + pass + +def delete_message(current): + pass + +def edit_message(current): + pass diff --git a/zengine/settings.py b/zengine/settings.py index 5fc8bad2..3c6fa042 100644 --- a/zengine/settings.py +++ b/zengine/settings.py @@ -115,6 +115,7 @@ VIEW_URLS = { 'dashboard': 'zengine.views.menu.Menu', 'ping': 'zengine.views.dev_utils.Ping', + '_zops_create_message': 'zengine.messaging.views.create_message', } From 2afff44bd5031c19feb4001f27a1a5471fc3693b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Mon, 27 Jun 2016 17:42:47 +0300 Subject: [PATCH 08/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/views.py | 48 ++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 21e8b28e..63c582f9 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -19,14 +19,15 @@ def create_message(current): """ Creates a message for the given channel. - API: - .. code-block:: python - {'view':'_zops_create_message', + # request: + { + 'view':'_zops_create_message', 'message': { 'channel': "code_name of the channel", 'receiver': "Key of receiver. Can be blank for non-direct messages", + 'client_id': "Client side unique id for referencing this message", 'title': "Title of the message. Can be blank.", 'body': "Message body.", 'type': zengine.messaging.model.MSG_TYPES, @@ -35,6 +36,10 @@ def create_message(current): 'name': "File name with extension.", 'content': "base64 encoded file content" }]} + # response: + { + 'msg_key': "Key of the just created message object", + } """ msg = current.input['message'] @@ -50,18 +55,47 @@ def create_message(current): def _dedect_file_type(current, name, content): # TODO: Attachment type detection - return 1 # Document + return 1 # Return as Document for now -def show_channel(current): +def show_public_channel(current): """ - Initial display of channel content + Initial display of channel content. + Returns chanel description, no of members, last 20 messages etc. - API: + .. code-block:: python + # request: + { + 'view':'_zops_show_public_channel', + 'channel_key': "Key of the requested channel" + } + + # response: + { + 'channel_key': "key of channel", + 'description': string, + 'no_of_members': int, + 'member_list': [ + {'name': string, + 'is_online': bool, + 'avatar_url': string, + }], + 'last_messages': [ + {'content': string, + 'key': string, + 'actions':[ + {'title': string, + 'cmd': string + } + ] + } + ] + } """ + def list_channels(current): pass From 40400869d0bdb0d6d028df0b2a8ba1f664c9be1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Tue, 28 Jun 2016 03:03:48 +0300 Subject: [PATCH 09/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/lib.py | 21 ++++++++ zengine/messaging/model.py | 49 +++++++++++++++-- zengine/messaging/views.py | 71 ++++++++++++++++++------- zengine/settings.py | 3 +- zengine/tornado_server/queue_manager.py | 7 +++ zengine/views/auth.py | 4 ++ 6 files changed, 130 insertions(+), 25 deletions(-) diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py index ac62a893..597ff0e7 100644 --- a/zengine/messaging/lib.py +++ b/zengine/messaging/lib.py @@ -13,8 +13,21 @@ from pyoko.conf import settings from zengine.client_queue import BLOCKING_MQ_PARAMS +from zengine.lib.cache import Cache +class ConnectionStatus(Cache): + """ + Cache object for workflow instances. + + Args: + wf_token: Token of the workflow instance. + """ + PREFIX = 'ONOFF' + + def __init__(self, user_id): + super(ConnectionStatus, self).__init__(user_id) + class BaseUser(object): connection = None @@ -48,6 +61,14 @@ def set_password(self, raw_password): self.password = pbkdf2_sha512.encrypt(raw_password, rounds=10000, salt_size=10) + def is_online(self, status=None): + if status is None: + return ConnectionStatus(self.key).get() + ConnectionStatus(self.key).set(status) + if status == False: + pass + # TODO: do + def pre_save(self): """ encrypt password if not already encrypted """ if self.password and not self.password.startswith('$pbkdf2'): diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index aa6b9e06..1bb36a99 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -84,8 +84,8 @@ def get_or_create_direct_channel(cls, initiator, receiver): else: channel_name = '%s_%s' % (initiator.key, receiver.key) channel = cls(is_direct=True, code_name=channel_name).save() - Subscription(channel=channel, user=initiator).save() - Subscription(channel=channel, user=receiver).save() + Subscriber(channel=channel, user=initiator).save() + Subscriber(channel=channel, user=receiver).save() return channel def add_message(self, body, title=None, sender=None, url=None, typ=2, receiver=None): @@ -95,6 +95,10 @@ def add_message(self, body, title=None, sender=None, url=None, typ=2, receiver=N return Message(sender=sender, body=body, msg_title=title, url=url, typ=typ, channel=self, receiver=receiver).save() + def get_last_messages(self): + # TODO: Refactor this with RabbitMQ Last Cached Messages exchange + return self.message_set.objects.filter()[:20] + @classmethod def _connect_mq(cls): if cls.connection is None or cls.connection.is_closed: @@ -123,13 +127,13 @@ def post_creation(self): self.create_exchange() -class Subscription(Model): +class Subscriber(Model): """ Permission model """ channel = Channel() - user = UserModel(reverse_name='channels') + user = UserModel(reverse_name='subscriptions') is_muted = field.Boolean("Mute the channel") inform_me = field.Boolean("Inform when I'm mentioned", default=True) visible = field.Boolean("Show under user's channel list", default=True) @@ -143,6 +147,10 @@ def _connect_mq(cls): cls.connection, cls.channel = get_mq_connection() return cls.channel + def unread_count(self): + # FIXME: track and return actual unread message count + return 0 + def create_exchange(self): """ Creates user's private exchange @@ -200,6 +208,32 @@ class Message(Model): body = field.String("Body") url = field.String("URL") + def get_actions_for(self, user): + actions = [ + ('Favorite', 'favorite_message') + ] + if self.sender == user: + actions.extend([ + ('Delete', 'delete_message'), + ('Edit', 'delete_message') + ]) + else: + actions.extend([ + ('Flag', 'flag_message') + ]) + + def serialize_for(self, user): + return { + 'content': self.body, + 'type': self.typ, + 'attachments': [attachment.serialize() for attachment in self.attachment_set], + 'title': self.msg_title, + 'sender_name': self.sender.full_name, + 'sender_key': self.sender.key, + 'key': self.key, + 'actions': self.get_actions_for(user), + } + def __unicode__(self): content = self.msg_title or self.body return "%s%s" % (content[:30], '...' if len(content) > 30 else '') @@ -225,6 +259,13 @@ class Attachment(Model): channel = Channel() message = Message() + def serialize(self): + return { + 'description': self.description, + 'file_name': self.name, + 'url': "%s%s" % (settings.S3_PUBLIC_URL, self.file) + } + def __unicode__(self): return self.name diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 63c582f9..07bbfb25 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -14,20 +14,18 @@ UserModel = get_object_from_path(settings.USER_MODEL) - def create_message(current): """ Creates a message for the given channel. .. code-block:: python - # request: + # request: { 'view':'_zops_create_message', 'message': { 'channel': "code_name of the channel", - 'receiver': "Key of receiver. Can be blank for non-direct messages", - 'client_id': "Client side unique id for referencing this message", + 'receiver': key, " of receiver. Should be set only for direct messages", 'title': "Title of the message. Can be blank.", 'body': "Message body.", 'type': zengine.messaging.model.MSG_TYPES, @@ -38,7 +36,7 @@ def create_message(current): }]} # response: { - 'msg_key': "Key of the just created message object", + 'msg_key': key, # of the just created message object, } """ @@ -53,14 +51,16 @@ def create_message(current): Attachment(channel=ch, msg=msg_obj, name=atch['name'], file=atch['content'], description=atch['description'], typ=typ).save() -def _dedect_file_type(current, name, content): - # TODO: Attachment type detection + +def _dedect_file_type(name, content): + # FIXME: Implement attachment type detection return 1 # Return as Document for now -def show_public_channel(current): + +def show_channel(current): """ Initial display of channel content. - Returns chanel description, no of members, last 20 messages etc. + Returns channel description, members, no of members, last 20 messages etc. .. code-block:: python @@ -68,12 +68,12 @@ def show_public_channel(current): # request: { 'view':'_zops_show_public_channel', - 'channel_key': "Key of the requested channel" + 'channel_key': key, } # response: { - 'channel_key': "key of channel", + 'channel_key': key, 'description': string, 'no_of_members': int, 'member_list': [ @@ -83,25 +83,45 @@ def show_public_channel(current): }], 'last_messages': [ {'content': string, - 'key': string, - 'actions':[ - {'title': string, - 'cmd': string - } - ] + 'title': string, + 'channel_key': key, + 'sender_name': string, + 'sender_key': key, + 'type': int, + 'key': key, + 'actions':[('name_string', 'cmd_string'),] } ] } """ - - + ch_key = current.input['channel_key'] + ch = Channel.objects.get(ch_key) + current.output = {'channel_key': ch_key, + 'description': ch.description, + 'no_of_members': len(ch.subscriber_set), + 'member_list': [{'name': sb.user.full_name, + 'is_online': sb.user.is_online(), + 'avatar_url': sb.user.get_avatar_url() + } for sb in ch.subscriber_set], + 'last_messages': [msg.serialize_for(current.user) + for msg in ch.get_last_messages()] + } + +def mark_offline_user(current): + current.user.is_online(False) def list_channels(current): - pass + return [ + {'name': sbs.channel.name, + 'key': sbs.channel.key, + 'unread': sbs.unread_count()} for sbs in + current.user.subscriptions] + def create_public_channel(current): pass + def create_direct_channel(current): """ Create a One-To-One channel for current user and selected user. @@ -109,11 +129,22 @@ def create_direct_channel(current): """ pass + +def create_broadcast_channel(current): + """ + Create a One-To-One channel for current user and selected user. + + """ + pass + + def find_message(current): pass + def delete_message(current): pass + def edit_message(current): pass diff --git a/zengine/settings.py b/zengine/settings.py index 3c6fa042..c3d0cf0f 100644 --- a/zengine/settings.py +++ b/zengine/settings.py @@ -116,7 +116,8 @@ 'dashboard': 'zengine.views.menu.Menu', 'ping': 'zengine.views.dev_utils.Ping', '_zops_create_message': 'zengine.messaging.views.create_message', - + 'mark_offline_user': 'zengine.messaging.views.mark_offline_user', + 'show_channel': 'zengine.messaging.views.show_channel', } if DEBUG: diff --git a/zengine/tornado_server/queue_manager.py b/zengine/tornado_server/queue_manager.py index a35427d3..d06b8878 100644 --- a/zengine/tornado_server/queue_manager.py +++ b/zengine/tornado_server/queue_manager.py @@ -186,8 +186,15 @@ def register_websocket(self, sess_id, ws): self.websockets[sess_id] = ws channel = self.create_out_channel(sess_id) + def inform_disconnection(self, sess_id): + self.websockets[sess_id].write_message({ + 'view': 'mark_offline_user', + 'sess_id': sess_id + }) + def unregister_websocket(self, sess_id): try: + self.inform_disconnection(sess_id) del self.websockets[sess_id] except KeyError: log.exception("Non-existent websocket") diff --git a/zengine/views/auth.py b/zengine/views/auth.py index 8afa7ce1..cc3dfc0f 100644 --- a/zengine/views/auth.py +++ b/zengine/views/auth.py @@ -64,6 +64,9 @@ def _do_binding(self): # routing_key="#" ) + def _user_is_online(self): + self.current.user.is_online(True) + def do_view(self): """ Authenticate user with given credentials. @@ -79,6 +82,7 @@ def do_view(self): self.current.input['password']) self.current.task_data['login_successful'] = auth_result if auth_result: + self._user_is_online() self._do_binding() user_sess = UserSessionID(self.current.user_id) old_sess_id = user_sess.get() From 70b3279a10b16a619fc237e9b678c9ecbedb7db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Tue, 28 Jun 2016 11:15:21 +0300 Subject: [PATCH 10/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/model.py | 28 ++++++++++++++++++------ zengine/messaging/views.py | 44 +++++++++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 1bb36a99..dd750293 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -88,12 +88,13 @@ def get_or_create_direct_channel(cls, initiator, receiver): Subscriber(channel=channel, user=receiver).save() return channel - def add_message(self, body, title=None, sender=None, url=None, typ=2, receiver=None): + @classmethod + def add_message(self, channel_key, body, title=None, sender=None, url=None, typ=2, receiver=None): mq_channel = self._connect_mq() mq_msg = json.dumps(dict(sender=sender, body=body, msg_title=title, url=url, typ=typ)) - mq_channel.basic_publish(exchange=self.code_name, body=mq_msg) + mq_channel.basic_publish(exchange=channel_key, body=mq_msg) return Message(sender=sender, body=body, msg_title=title, url=url, - typ=typ, channel=self, receiver=receiver).save() + typ=typ, channel_id=channel_key, receiver=receiver).save() def get_last_messages(self): # TODO: Refactor this with RabbitMQ Last Cached Messages exchange @@ -117,11 +118,15 @@ def pre_creation(self): if not self.code_name: if self.name: self.code_name = to_safe_str(self.name) + self.key = self.code_name return if self.owner and self.is_private: self.code_name = "prv_%s" % to_safe_str(self.owner.key) + self.key = self.code_name return raise IntegrityError('Non-private and non-direct channels should have a "name".') + else: + self.key = self.code_name def post_creation(self): self.create_exchange() @@ -138,6 +143,7 @@ class Subscriber(Model): inform_me = field.Boolean("Inform when I'm mentioned", default=True) visible = field.Boolean("Show under user's channel list", default=True) can_leave = field.Boolean("Membership is not obligatory", default=True) + last_seen = field.DateTime("Last seen time") # status = field.Integer("Status", choices=SUBSCRIPTION_STATUS) @@ -161,6 +167,10 @@ def create_exchange(self): channel = self._connect_mq() channel.exchange_declare(exchange=self.user.key, exchange_type='fanout', durable=True) + @classmethod + def mark_seen(cls, key, datetime_str): + cls.objects.filter(key=key).update(last_seen=datetime_str) + def bind_to_channel(self): """ Binds (subscribes) users private exchange to channel exchange @@ -197,13 +207,18 @@ def __unicode__(self): class Message(Model): """ - Permission model + Message model + + Notes: + Never use directly for creating new messages! Use these methods: + - Channel objects's **add_message()** method. + - User object's **set_message()** method. (which uses channel.add_message) """ channel = Channel() sender = UserModel(reverse_name='sent_messages') receiver = UserModel(reverse_name='received_messages') - typ = field.Integer("Type", choices=MSG_TYPES) - status = field.Integer("Status", choices=MESSAGE_STATUS) + typ = field.Integer("Type", choices=MSG_TYPES, default=1) + status = field.Integer("Status", choices=MESSAGE_STATUS, default=1) msg_title = field.String("Title") body = field.String("Body") url = field.String("URL") @@ -226,6 +241,7 @@ def serialize_for(self, user): return { 'content': self.body, 'type': self.typ, + 'time': self.updated_at, 'attachments': [attachment.serialize() for attachment in self.attachment_set], 'title': self.msg_title, 'sender_name': self.sender.full_name, diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 07bbfb25..7d520304 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -24,7 +24,7 @@ def create_message(current): { 'view':'_zops_create_message', 'message': { - 'channel': "code_name of the channel", + 'channel': key, # of channel", 'receiver': key, " of receiver. Should be set only for direct messages", 'title': "Title of the message. Can be blank.", 'body': "Message body.", @@ -84,6 +84,7 @@ def show_channel(current): 'last_messages': [ {'content': string, 'title': string, + 'time': datetime, 'channel_key': key, 'sender_name': string, 'sender_key': key, @@ -107,6 +108,47 @@ def show_channel(current): for msg in ch.get_last_messages()] } +def last_seen_channel(current): + """ + Initial display of channel content. + Returns channel description, members, no of members, last 20 messages etc. + + + .. code-block:: python + + # request: + { + 'view':'_zops_last_seen_msg', + 'channel_key': key, + 'msg_key': key, + 'msg_date': datetime, + } + + # response: + { + 'channel_key': key, + 'description': string, + 'no_of_members': int, + 'member_list': [ + {'name': string, + 'is_online': bool, + 'avatar_url': string, + }], + 'last_messages': [ + {'content': string, + 'title': string, + 'channel_key': key, + 'sender_name': string, + 'sender_key': key, + 'type': int, + 'key': key, + 'actions':[('name_string', 'cmd_string'),] + } + ] + } + """ + current.input['seen_channel'] + def mark_offline_user(current): current.user.is_online(False) From 5b712ab7abff0e732e8b86d9bbcb64bd245985af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Wed, 29 Jun 2016 00:27:54 +0300 Subject: [PATCH 11/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/client_queue.py | 4 +-- zengine/messaging/lib.py | 41 ++++++++++++------------- zengine/messaging/model.py | 19 ++++++------ zengine/tornado_server/queue_manager.py | 3 +- zengine/views/auth.py | 18 +++-------- zengine/wf_daemon.py | 14 +++++---- 6 files changed, 46 insertions(+), 53 deletions(-) diff --git a/zengine/client_queue.py b/zengine/client_queue.py index 6f0f0ea6..e4907805 100644 --- a/zengine/client_queue.py +++ b/zengine/client_queue.py @@ -54,8 +54,8 @@ def get_sess_id(self): return self.sess_id def send_to_queue(self, message=None, json_message=None): - self.get_channel().basic_publish(exchange='', - routing_key=self.get_sess_id(), + self.get_channel().basic_publish(exchange=self.user_id or '', + routing_key=self.sess_id, body=json_message or json.dumps(message)) def old_to_new_queue(self, old_sess_id): diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py index 597ff0e7..3349b7e9 100644 --- a/zengine/messaging/lib.py +++ b/zengine/messaging/lib.py @@ -30,14 +30,14 @@ def __init__(self, user_id): class BaseUser(object): - connection = None - channel = None + mq_connection = None + mq_channel = None def _connect_mq(self): - if not self.connection is None or self.connection.is_closed: - self.connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) - self.channel = self.connection.channel() - return self.channel + if not self.mq_connection is None or self.mq_connection.is_closed: + self.mq_connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) + self.mq_channel = self.mq_connection.channel() + return self.mq_channel def get_avatar_url(self): """ @@ -90,7 +90,7 @@ def check_password(self, raw_password): def get_role(self, role_id): """ - Kullanıcıya ait Role nesnesini getirir. + Retrieves user's roles. Args: role_id (int) @@ -105,7 +105,11 @@ def get_role(self, role_id): def full_name(self): return self.username - def send_message(self, title, message, sender=None, url=None, typ=1): + def bind_private_channel(self, sess_id): + mq_channel = self._connect_mq() + mq_channel.queue_bind(exchange='prv_%s' % self.key, queue=sess_id) + + def send_notification(self, title, message, typ=1, url=None): """ sends message to users private mq exchange Args: @@ -117,17 +121,10 @@ def send_message(self, title, message, sender=None, url=None, typ=1): """ - mq_channel = self._connect_mq() - mq_msg = dict(body=message, msg_title=title, url=url, typ=typ) - if sender: - mq_msg['sender_name'] = sender.full_name - mq_msg['sender_key'] = sender.key - - mq_channel.basic_publish(exchange=self.key, body=json.dumps(mq_msg)) - self._write_message_to_db(sender, message, title, url, typ) - - def _write_message_to_db(self, sender, body, title, url, typ): - from zengine.messaging.model import Channel, Message - channel = Channel.objects.get(owner=self, is_private=True) - Message(channel=channel, sender=sender, msg_title=title, - body=body, receiver=self, url=url, typ=typ).save() + self.channel_set.channel.__class__.add_message( + channel_key='prv_%s' % self.key, + body=message, + title=title, + typ=typ, + url=url + ) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index dd750293..f2e734af 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -44,8 +44,8 @@ class Channel(Model): is_direct: Represents a user-to-user direct message exchange """ - channel = None - connection = None + mq_channel = None + mq_connection = None name = field.String("Name") code_name = field.String("Internal name") @@ -62,7 +62,8 @@ class Meta: unique_together = (('is_private', 'owner'),) class Managers(ListNode): - user = UserModel(reverse_name='managed_channels') + user = UserModel() + @classmethod def get_or_create_direct_channel(cls, initiator, receiver): @@ -102,17 +103,17 @@ def get_last_messages(self): @classmethod def _connect_mq(cls): - if cls.connection is None or cls.connection.is_closed: - cls.connection, cls.channel = get_mq_connection() - return cls.channel + if cls.mq_connection is None or cls.mq_connection.is_closed: + cls.mq_connection, cls.mq_channel = get_mq_connection() + return cls.mq_channel def create_exchange(self): """ Creates MQ exchange for this channel Needs to be defined only once. """ - channel = self._connect_mq() - channel.exchange_declare(exchange=self.code_name, exchange_type='fanout', durable=True) + mq_channel = self._connect_mq() + mq_channel.exchange_declare(exchange=self.code_name, exchange_type='fanout', durable=True) def pre_creation(self): if not self.code_name: @@ -212,7 +213,7 @@ class Message(Model): Notes: Never use directly for creating new messages! Use these methods: - Channel objects's **add_message()** method. - - User object's **set_message()** method. (which uses channel.add_message) + - User object's **set_message()** method. (which also uses channel.add_message) """ channel = Channel() sender = UserModel(reverse_name='sent_messages') diff --git a/zengine/tornado_server/queue_manager.py b/zengine/tornado_server/queue_manager.py index d06b8878..7df79801 100644 --- a/zengine/tornado_server/queue_manager.py +++ b/zengine/tornado_server/queue_manager.py @@ -51,7 +51,7 @@ class BlockingConnectionForHTTP(object): - REPLY_TIMEOUT = 10 # sec + REPLY_TIMEOUT = 100 # sec def __init__(self): self.connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) @@ -216,6 +216,7 @@ def _on_output_queue_decleration(queue): # exclusive=True ) + self.connection.channel(_on_output_channel_creation) def redirect_incoming_message(self, sess_id, message, request): diff --git a/zengine/views/auth.py b/zengine/views/auth.py index cc3dfc0f..f84c9f8d 100644 --- a/zengine/views/auth.py +++ b/zengine/views/auth.py @@ -53,17 +53,6 @@ class Login(SimpleView): does the authentication at ``do`` stage. """ - def _do_binding(self): - """ - Bind user's ephemeral session queue to user's durable private exchange - """ - from zengine.messaging.model import get_mq_connection - connection, channel = get_mq_connection() - channel.queue_bind(exchange=self.current.user_id, - queue=self.current.session.sess_id, - # routing_key="#" - ) - def _user_is_online(self): self.current.user.is_online(True) @@ -72,6 +61,7 @@ def do_view(self): Authenticate user with given credentials. Connects user's queue and exchange """ + self.current.output['login_process'] = True self.current.task_data['login_successful'] = False if self.current.is_auth: self.current.output['cmd'] = 'upgrade' @@ -82,8 +72,8 @@ def do_view(self): self.current.input['password']) self.current.task_data['login_successful'] = auth_result if auth_result: - self._user_is_online() - self._do_binding() + self.current.user.is_online(True) + self.current.user.bind_private_channel(self.current.session.sess_id) user_sess = UserSessionID(self.current.user_id) old_sess_id = user_sess.get() user_sess.set(self.current.session.sess_id) @@ -104,7 +94,9 @@ def show_view(self): """ Show :attr:`LoginForm` form. """ + self.current.output['login_process'] = True if self.current.is_auth: self.current.output['cmd'] = 'upgrade' else: + self.current.output['forms'] = LoginForm(current=self.current).serialize() diff --git a/zengine/wf_daemon.py b/zengine/wf_daemon.py index 4002bda3..d481f56a 100755 --- a/zengine/wf_daemon.py +++ b/zengine/wf_daemon.py @@ -62,10 +62,13 @@ def connect(self): self.client_queue = ClientQueue() self.input_channel = self.connection.channel() - self.input_channel.exchange_declare(exchange=self.INPUT_EXCHANGE, type='topic', durable=True) + self.input_channel.exchange_declare(exchange=self.INPUT_EXCHANGE, + type='topic', + durable=True) self.input_channel.queue_declare(queue=self.INPUT_QUEUE_NAME) self.input_channel.queue_bind(exchange=self.INPUT_EXCHANGE, queue=self.INPUT_QUEUE_NAME) - log.info("Bind to queue named '%s' queue with exchange '%s'" % (self.INPUT_QUEUE_NAME, self.INPUT_EXCHANGE)) + log.info("Bind to queue named '%s' queue with exchange '%s'" % (self.INPUT_QUEUE_NAME, + self.INPUT_EXCHANGE)) def run(self): """ @@ -172,11 +175,10 @@ def handle_message(self, ch, method, properties, body): def send_output(self, output, sessid): self.client_queue.sess_id = sessid + # TODO: This is ugly + if 'login_process' not in output: + self.client_queue.user_id = self.current.user_id self.client_queue.send_to_queue(output) - # self.output_channel.basic_publish(exchange='', - # routing_key=sessid, - # body=json.dumps(output)) - # except ConnectionClosed: def run_workers(no_subprocess): From 0c17f7a8e2f5faa2c03435dc04ca6a0e67862999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Wed, 29 Jun 2016 02:01:35 +0300 Subject: [PATCH 12/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/client_queue.py | 6 ++++-- zengine/messaging/lib.py | 11 ++++++----- zengine/tornado_server/queue_manager.py | 16 +++++++++------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/zengine/client_queue.py b/zengine/client_queue.py index e4907805..50046a1a 100644 --- a/zengine/client_queue.py +++ b/zengine/client_queue.py @@ -22,7 +22,7 @@ heartbeat_interval=0, credentials=pika.PlainCredentials(settings.MQ_USER, settings.MQ_PASS) ) - +from zengine.log import log class ClientQueue(object): """ @@ -54,7 +54,9 @@ def get_sess_id(self): return self.sess_id def send_to_queue(self, message=None, json_message=None): - self.get_channel().basic_publish(exchange=self.user_id or '', + log.debug("Sending following message to %s queue:\n%s " % ( + self.sess_id, json_message or json.dumps(message))) + self.get_channel().publish(exchange=self.user_id or '', routing_key=self.sess_id, body=json_message or json.dumps(message)) diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py index 3349b7e9..4b85c975 100644 --- a/zengine/messaging/lib.py +++ b/zengine/messaging/lib.py @@ -33,11 +33,12 @@ class BaseUser(object): mq_connection = None mq_channel = None - def _connect_mq(self): - if not self.mq_connection is None or self.mq_connection.is_closed: - self.mq_connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) - self.mq_channel = self.mq_connection.channel() - return self.mq_channel + @classmethod + def _connect_mq(cls): + if cls.mq_connection is None or cls.mq_connection.is_closed: + cls.mq_connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) + cls.mq_channel = cls.mq_connection.channel() + return cls.mq_channel def get_avatar_url(self): """ diff --git a/zengine/tornado_server/queue_manager.py b/zengine/tornado_server/queue_manager.py index 7df79801..d490eeda 100644 --- a/zengine/tornado_server/queue_manager.py +++ b/zengine/tornado_server/queue_manager.py @@ -51,7 +51,7 @@ class BlockingConnectionForHTTP(object): - REPLY_TIMEOUT = 100 # sec + REPLY_TIMEOUT = 5 # sec def __init__(self): self.connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) @@ -76,6 +76,7 @@ def _wait_for_reply(self, sess_id, input_data): timeout_start = time.time() while 1: method_frame, header_frame, body = channel.basic_get(sess_id) + log.debug("\n%s\n%s\n%s\n%s" % (sess_id, method_frame, header_frame, body)) if method_frame: reply = json_decode(body) if 'callbackID' in reply and reply['callbackID'] == input_data['callbackID']: @@ -90,7 +91,7 @@ def _wait_for_reply(self, sess_id, input_data): if time.time() - timeout_start > self.REPLY_TIMEOUT: break else: - time.sleep(0.4) + time.sleep(1) log.info('No message returned for %s' % sess_id) channel.close() @@ -187,10 +188,12 @@ def register_websocket(self, sess_id, ws): channel = self.create_out_channel(sess_id) def inform_disconnection(self, sess_id): - self.websockets[sess_id].write_message({ - 'view': 'mark_offline_user', - 'sess_id': sess_id - }) + self.in_channel.basic_publish(exchange='input_exc', + routing_key=sess_id, + body=json_encode({ + 'view': 'mark_offline_user', + 'sess_id': sess_id + })) def unregister_websocket(self, sess_id): try: @@ -216,7 +219,6 @@ def _on_output_queue_decleration(queue): # exclusive=True ) - self.connection.channel(_on_output_channel_creation) def redirect_incoming_message(self, sess_id, message, request): From b3f8e53d2f005e2ec78e93985eb06e88df78d9c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Wed, 29 Jun 2016 02:32:01 +0300 Subject: [PATCH 13/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/client_queue.py | 8 +++++--- zengine/messaging/lib.py | 3 ++- zengine/tornado_server/queue_manager.py | 4 ++-- zengine/wf_daemon.py | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/zengine/client_queue.py b/zengine/client_queue.py index 50046a1a..bf0d8536 100644 --- a/zengine/client_queue.py +++ b/zengine/client_queue.py @@ -54,9 +54,11 @@ def get_sess_id(self): return self.sess_id def send_to_queue(self, message=None, json_message=None): - log.debug("Sending following message to %s queue:\n%s " % ( - self.sess_id, json_message or json.dumps(message))) - self.get_channel().publish(exchange=self.user_id or '', + exchange = self.user_id or '' + log.debug("Sending following message to %s queue, \"%s\" exchange:\n%s " % ( + self.sess_id, exchange, json_message or json.dumps(message))) + + self.get_channel().publish(exchange=exchange, routing_key=self.sess_id, body=json_message or json.dumps(message)) diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py index 4b85c975..8e3a97b6 100644 --- a/zengine/messaging/lib.py +++ b/zengine/messaging/lib.py @@ -106,7 +106,8 @@ def get_role(self, role_id): def full_name(self): return self.username - def bind_private_channel(self, sess_id): + @classmethod + def bind_private_channel(cls, sess_id): mq_channel = self._connect_mq() mq_channel.queue_bind(exchange='prv_%s' % self.key, queue=sess_id) diff --git a/zengine/tornado_server/queue_manager.py b/zengine/tornado_server/queue_manager.py index d490eeda..841a5736 100644 --- a/zengine/tornado_server/queue_manager.py +++ b/zengine/tornado_server/queue_manager.py @@ -190,10 +190,10 @@ def register_websocket(self, sess_id, ws): def inform_disconnection(self, sess_id): self.in_channel.basic_publish(exchange='input_exc', routing_key=sess_id, - body=json_encode({ + body=json_encode(dict(data={ 'view': 'mark_offline_user', 'sess_id': sess_id - })) + }))) def unregister_websocket(self, sess_id): try: diff --git a/zengine/wf_daemon.py b/zengine/wf_daemon.py index d481f56a..4ddd5694 100755 --- a/zengine/wf_daemon.py +++ b/zengine/wf_daemon.py @@ -166,7 +166,7 @@ def handle_message(self, ch, method, properties, body): raise err = traceback.format_exc() output = {'error': self._prepare_error_msg(err), "code": 500} - log.exception("Worker error occurred") + log.exception("Worker error occurred with messsage body:\n%s" % body) if 'callbackID' in input: output['callbackID'] = input['callbackID'] log.info("OUTPUT for %s: %s" % (sessid, output)) From 864d1724b8ed0f4cf786693a903bdd75ddb59f0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Thu, 30 Jun 2016 00:05:52 +0300 Subject: [PATCH 14/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- requirements/local_dev.txt | 5 +- zengine/client_queue.py | 79 +++++++++++++------------ zengine/lib/test_utils.py | 3 +- zengine/messaging/lib.py | 18 ++++-- zengine/messaging/model.py | 18 ++++-- zengine/tornado_server/get_logger.py | 9 ++- zengine/tornado_server/queue_manager.py | 28 ++++++--- zengine/views/auth.py | 31 ++++++---- zengine/wf_daemon.py | 27 +++++---- 9 files changed, 130 insertions(+), 88 deletions(-) diff --git a/requirements/local_dev.txt b/requirements/local_dev.txt index e8fa70a4..ea222090 100644 --- a/requirements/local_dev.txt +++ b/requirements/local_dev.txt @@ -10,6 +10,5 @@ lazy_object_proxy enum34 werkzeug pytest -celery -werkzeug -pytest +pika +tornado diff --git a/zengine/client_queue.py b/zengine/client_queue.py index bf0d8536..fbbd4d8a 100644 --- a/zengine/client_queue.py +++ b/zengine/client_queue.py @@ -30,10 +30,10 @@ class ClientQueue(object): """ def __init__(self, user_id=None, sess_id=None): - self.user_id = user_id + # self.user_id = user_id self.connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) self.channel = self.connection.channel() - self.sess_id = sess_id + # self.sess_id = sess_id def close(self): self.channel.close() @@ -47,42 +47,45 @@ def get_channel(self): self.connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) self.channel = pika.BlockingConnection(BLOCKING_MQ_PARAMS) return self.channel + # + # def get_sess_id(self): + # if not self.sess_id: + # self.sess_id = UserSessionID(self.user_id).get() + # return self.sess_id - def get_sess_id(self): - if not self.sess_id: - self.sess_id = UserSessionID(self.user_id).get() - return self.sess_id + def send_to_default_exchange(self, sess_id, message=None): + msg = json.dumps(message) + log.debug("Sending following message to %s queue through default exchange:\n%s" % ( + sess_id, msg)) + self.get_channel().publish(exchange='', routing_key=sess_id, body=msg) - def send_to_queue(self, message=None, json_message=None): - exchange = self.user_id or '' - log.debug("Sending following message to %s queue, \"%s\" exchange:\n%s " % ( - self.sess_id, exchange, json_message or json.dumps(message))) + def send_to_prv_exchange(self, user_id, message=None): + exchange = 'prv_%s' % user_id.lower() + msg = json.dumps(message) + log.debug("Sending following users \"%s\" exchange:\n%s " % (exchange, msg)) + self.get_channel().publish(exchange=exchange, routing_key='', body=msg) - self.get_channel().publish(exchange=exchange, - routing_key=self.sess_id, - body=json_message or json.dumps(message)) - - def old_to_new_queue(self, old_sess_id): - """ - Somehow if users old (obsolete) queue has - undelivered messages, we should redirect them to - current queue. - """ - old_input_channel = self.connection.channel() - while True: - try: - method_frame, header_frame, body = old_input_channel.basic_get(old_sess_id) - if method_frame: - self.send_to_queue(json_message=body) - old_input_channel.basic_ack(method_frame.delivery_tag) - else: - old_input_channel.queue_delete(old_sess_id) - old_input_channel.close() - break - except ChannelClosed as e: - if e[0] == 404: - break - # e => (404, "NOT_FOUND - no queue 'sess_id' in vhost '/'") - else: - raise - # old_input_channel = self.connection.channel() + # def old_to_new_queue(self, old_sess_id): + # """ + # Somehow if users old (obsolete) queue has + # undelivered messages, we should redirect them to + # current queue. + # """ + # old_input_channel = self.connection.channel() + # while True: + # try: + # method_frame, header_frame, body = old_input_channel.basic_get(old_sess_id) + # if method_frame: + # self.send_to_queue(json_message=body) + # old_input_channel.basic_ack(method_frame.delivery_tag) + # else: + # old_input_channel.queue_delete(old_sess_id) + # old_input_channel.close() + # break + # except ChannelClosed as e: + # if e[0] == 404: + # break + # # e => (404, "NOT_FOUND - no queue 'sess_id' in vhost '/'") + # else: + # raise + # # old_input_channel = self.connection.channel() diff --git a/zengine/lib/test_utils.py b/zengine/lib/test_utils.py index bcb24288..8fa2299f 100644 --- a/zengine/lib/test_utils.py +++ b/zengine/lib/test_utils.py @@ -126,7 +126,7 @@ def post(self, **data): self.token = self.response_wrapper.token return self.response_wrapper - def send_output(self, output, sessid): + def send_output(self, output): self.response_wrapper = ResponseWrapper(output) @@ -219,6 +219,7 @@ def _do_login(self): assert all([(field in req_fields) for field in ('username', 'password')]) resp = self.client.post(username=self.client.username or self.client.user.username, password="123", cmd="do") + log.debug("login result :\n%s" % resp.json) assert resp.json['cmd'] == 'upgrade' @staticmethod diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py index 8e3a97b6..3f397b60 100644 --- a/zengine/messaging/lib.py +++ b/zengine/messaging/lib.py @@ -14,7 +14,7 @@ from pyoko.conf import settings from zengine.client_queue import BLOCKING_MQ_PARAMS from zengine.lib.cache import Cache - +from zengine.log import log class ConnectionStatus(Cache): """ @@ -106,10 +106,16 @@ def get_role(self, role_id): def full_name(self): return self.username - @classmethod - def bind_private_channel(cls, sess_id): - mq_channel = self._connect_mq() - mq_channel.queue_bind(exchange='prv_%s' % self.key, queue=sess_id) + @property + def prv_exchange(self): + return 'prv_%s' % self.key.lower() + + def bind_private_channel(self, sess_id): + mq_channel = pika.BlockingConnection(BLOCKING_MQ_PARAMS).channel() + mq_channel.queue_declare(queue=sess_id, arguments={'x-expires': 40000}) + log.debug("Binding private exchange to client queue: Q:%s --> E:%s" % (sess_id, + self.prv_exchange)) + mq_channel.queue_bind(exchange=self.prv_exchange, queue=sess_id) def send_notification(self, title, message, typ=1, url=None): """ @@ -124,7 +130,7 @@ def send_notification(self, title, message, typ=1, url=None): """ self.channel_set.channel.__class__.add_message( - channel_key='prv_%s' % self.key, + channel_key=self.prv_exchange, body=message, title=title, typ=typ, diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index f2e734af..b69c4cd9 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -113,7 +113,9 @@ def create_exchange(self): Needs to be defined only once. """ mq_channel = self._connect_mq() - mq_channel.exchange_declare(exchange=self.code_name, exchange_type='fanout', durable=True) + mq_channel.exchange_declare(exchange=self.code_name, + exchange_type='fanout', + durable=True) def pre_creation(self): if not self.code_name: @@ -122,7 +124,7 @@ def pre_creation(self): self.key = self.code_name return if self.owner and self.is_private: - self.code_name = "prv_%s" % to_safe_str(self.owner.key) + self.code_name = self.owner.prv_exchange self.key = self.code_name return raise IntegrityError('Non-private and non-direct channels should have a "name".') @@ -161,12 +163,16 @@ def unread_count(self): def create_exchange(self): """ Creates user's private exchange - Actually needed to be defined only once. - but since we don't know if it's exists or not - we always call it before binding it to related channel + + Actually user's private channel needed to be defined only once, + and this should be happened when user first created. + But since this has a little performance cost, + to be safe we always call it before binding to the channel we currently subscribe """ channel = self._connect_mq() - channel.exchange_declare(exchange=self.user.key, exchange_type='fanout', durable=True) + channel.exchange_declare(exchange='prv_%s' % self.user.key.lower(), + exchange_type='fanout', + durable=True) @classmethod def mark_seen(cls, key, datetime_str): diff --git a/zengine/tornado_server/get_logger.py b/zengine/tornado_server/get_logger.py index 8fa1aa61..06e8cca8 100644 --- a/zengine/tornado_server/get_logger.py +++ b/zengine/tornado_server/get_logger.py @@ -23,8 +23,13 @@ def get_logger(settings): # ch.setLevel(logging.DEBUG) # create formatter - formatter = logging.Formatter( - '%(asctime)s - %(process)d - %(pathname)s:%(lineno)d [%(module)s > %(funcName)s] - %(name)s - %(levelname)s - %(message)s') + if settings.DEBUG: + # make log messages concise and readble for developemnt + format_str = '%(created)d - %(filename)s:%(lineno)d [%(module)s > %(funcName)s] - %(name)s - %(levelname)s - %(message)s' + else: + format_str = '%(asctime)s - %(process)d - %(pathname)s:%(lineno)d [%(module)s > %(funcName)s] - %(name)s - %(levelname)s - %(message)s' + + formatter = logging.Formatter(format_str) # add formatter to ch ch.setFormatter(formatter) diff --git a/zengine/tornado_server/queue_manager.py b/zengine/tornado_server/queue_manager.py index 841a5736..39b4bd05 100644 --- a/zengine/tornado_server/queue_manager.py +++ b/zengine/tornado_server/queue_manager.py @@ -9,8 +9,8 @@ import json from uuid import uuid4 -import os - +import os, sys +sys.sessid_to_userid = {} import pika import time from pika.adapters import TornadoConnection, BaseConnection @@ -30,6 +30,7 @@ 'MQ_PORT': int(os.environ.get('MQ_PORT', '5672')), 'MQ_USER': os.environ.get('MQ_USER', 'guest'), 'MQ_PASS': os.environ.get('MQ_PASS', 'guest'), + 'DEBUG': os.environ.get('DEBUG', False), 'MQ_VHOST': os.environ.get('MQ_VHOST', '/'), }) log = get_logger(settings) @@ -70,9 +71,15 @@ def _send_message(self, sess_id, input_data): routing_key=sess_id, body=json_encode(input_data)) + def _store_user_id(self, sess_id, body): + sys.sessid_to_userid[sess_id[5:]] = json_decode(body)['user_id'] + def _wait_for_reply(self, sess_id, input_data): channel = self.create_channel() - channel.queue_declare(queue=sess_id, auto_delete=True) + channel.queue_declare(queue=sess_id, + arguments={'x-expires': 4000} + # auto_delete=True + ) timeout_start = time.time() while 1: method_frame, header_frame, body = channel.basic_get(sess_id) @@ -83,6 +90,8 @@ def _wait_for_reply(self, sess_id, input_data): channel.basic_ack(method_frame.delivery_tag) channel.close() log.info('Returned view message for %s: %s' % (sess_id, body)) + if 'upgrade' in body: + self._store_user_id(sess_id, body) return body else: if time.time() - json_decode(body)['reply_timestamp'] > self.REPLY_TIMEOUT: @@ -184,6 +193,7 @@ def register_websocket(self, sess_id, ws): sess_id: ws: """ + user_id = sys.sessid_to_userid[sess_id] self.websockets[sess_id] = ws channel = self.create_out_channel(sess_id) @@ -192,8 +202,8 @@ def inform_disconnection(self, sess_id): routing_key=sess_id, body=json_encode(dict(data={ 'view': 'mark_offline_user', - 'sess_id': sess_id - }))) + 'sess_id': sess_id,}, + _zops_remote_ip=''))) def unregister_websocket(self, sess_id): try: @@ -211,16 +221,19 @@ def create_out_channel(self, sess_id): def _on_output_channel_creation(channel): def _on_output_queue_decleration(queue): channel.basic_consume(self.on_message, queue=sess_id) - + log.debug("BIND QUEUE TO WS Q.%s on Ch.%s" % (sess_id, channel.consumer_tags[0])) self.out_channels[sess_id] = channel + channel.queue_declare(callback=_on_output_queue_decleration, queue=sess_id, + arguments={'x-expires': 40000}, # auto_delete=True, # exclusive=True ) self.connection.channel(_on_output_channel_creation) + def redirect_incoming_message(self, sess_id, message, request): message = json_decode(message) message['_zops_sess_id'] = sess_id @@ -231,9 +244,10 @@ def redirect_incoming_message(self, sess_id, message, request): def on_message(self, channel, method, header, body): sess_id = method.routing_key + log.debug("WS RPLY for %s: %s" % (sess_id, body)) if sess_id in self.websockets: self.websockets[sess_id].write_message(body) - log.debug("WS RPLY for %s: %s" % (sess_id, body)) + channel.basic_ack(delivery_tag=method.delivery_tag) # else: # channel.basic_reject(delivery_tag=method.delivery_tag) diff --git a/zengine/views/auth.py b/zengine/views/auth.py index f84c9f8d..d8571fdf 100644 --- a/zengine/views/auth.py +++ b/zengine/views/auth.py @@ -56,6 +56,15 @@ class Login(SimpleView): def _user_is_online(self): self.current.user.is_online(True) + def _do_upgrade(self): + """ open websocket connection """ + self.current.output['cmd'] = 'upgrade' + self.current.output['user_id'] = self.current.user_id + self.current.user.is_online(True) + self.current.user.bind_private_channel(self.current.session.sess_id) + user_sess = UserSessionID(self.current.user_id) + user_sess.set(self.current.session.sess_id) + def do_view(self): """ Authenticate user with given credentials. @@ -64,7 +73,7 @@ def do_view(self): self.current.output['login_process'] = True self.current.task_data['login_successful'] = False if self.current.is_auth: - self.current.output['cmd'] = 'upgrade' + self._do_upgrade() else: try: auth_result = self.current.auth.authenticate( @@ -72,16 +81,13 @@ def do_view(self): self.current.input['password']) self.current.task_data['login_successful'] = auth_result if auth_result: - self.current.user.is_online(True) - self.current.user.bind_private_channel(self.current.session.sess_id) - user_sess = UserSessionID(self.current.user_id) - old_sess_id = user_sess.get() - user_sess.set(self.current.session.sess_id) - notify = Notify(self.current.user_id) - notify.cache_to_queue() - if old_sess_id: - notify.old_to_new_queue(old_sess_id) - self.current.output['cmd'] = 'upgrade' + self._do_upgrade() + + # old_sess_id = user_sess.get() + # notify = Notify(self.current.user_id) + # notify.cache_to_queue() + # if old_sess_id: + # notify.old_to_new_queue(old_sess_id) except: raise self.current.log.exception("Wrong username or another error occurred") @@ -96,7 +102,6 @@ def show_view(self): """ self.current.output['login_process'] = True if self.current.is_auth: - self.current.output['cmd'] = 'upgrade' + self._do_upgrade() else: - self.current.output['forms'] = LoginForm(current=self.current).serialize() diff --git a/zengine/wf_daemon.py b/zengine/wf_daemon.py index 4ddd5694..215b1271 100755 --- a/zengine/wf_daemon.py +++ b/zengine/wf_daemon.py @@ -132,7 +132,7 @@ def handle_message(self, ch, method, properties, body): """ input = {} try: - sessid = method.routing_key + self.sessid = method.routing_key input = json_decode(body) data = input['data'] @@ -144,9 +144,9 @@ def handle_message(self, ch, method, properties, body): data['view'] = data['path'] else: data['wf'] = data['path'] - session = Session(sessid[5:]) # clip "HTTP_" prefix from sessid + session = Session(self.sessid[5:]) # clip "HTTP_" prefix from sessid else: - session = Session(sessid) + session = Session(self.sessid) headers = {'remote_ip': input['_zops_remote_ip']} @@ -169,16 +169,19 @@ def handle_message(self, ch, method, properties, body): log.exception("Worker error occurred with messsage body:\n%s" % body) if 'callbackID' in input: output['callbackID'] = input['callbackID'] - log.info("OUTPUT for %s: %s" % (sessid, output)) + log.info("OUTPUT for %s: %s" % (self.sessid, output)) output['reply_timestamp'] = time() - self.send_output(output, sessid) - - def send_output(self, output, sessid): - self.client_queue.sess_id = sessid - # TODO: This is ugly - if 'login_process' not in output: - self.client_queue.user_id = self.current.user_id - self.client_queue.send_to_queue(output) + self.send_output(output) + + def send_output(self, output): + # TODO: This is ugly, we should separate login process + log.debug("SEND_OUTPUT: %s" % output) + if self.current.user_id is None or 'login_process' in output: + self.client_queue.send_to_default_exchange(self.sessid, output) + else: + self.client_queue.send_to_prv_exchange(self.current.user_id, output) + + def run_workers(no_subprocess): From cebba90118af5b18e1b5c1db09df969b8d61efeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Thu, 30 Jun 2016 00:35:25 +0300 Subject: [PATCH 15/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/tornado_server/queue_manager.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/zengine/tornado_server/queue_manager.py b/zengine/tornado_server/queue_manager.py index 39b4bd05..c9465c16 100644 --- a/zengine/tornado_server/queue_manager.py +++ b/zengine/tornado_server/queue_manager.py @@ -72,7 +72,7 @@ def _send_message(self, sess_id, input_data): body=json_encode(input_data)) def _store_user_id(self, sess_id, body): - sys.sessid_to_userid[sess_id[5:]] = json_decode(body)['user_id'] + sys.sessid_to_userid[sess_id[5:]] = json_decode(body)['user_id'].lower() def _wait_for_reply(self, sess_id, input_data): channel = self.create_channel() @@ -194,8 +194,8 @@ def register_websocket(self, sess_id, ws): ws: """ user_id = sys.sessid_to_userid[sess_id] - self.websockets[sess_id] = ws - channel = self.create_out_channel(sess_id) + self.websockets[user_id] = ws + self.create_out_channel(sess_id, user_id) def inform_disconnection(self, sess_id): self.in_channel.basic_publish(exchange='input_exc', @@ -206,22 +206,25 @@ def inform_disconnection(self, sess_id): _zops_remote_ip=''))) def unregister_websocket(self, sess_id): + user_id = sys.sessid_to_userid.get(sess_id, None) try: self.inform_disconnection(sess_id) - del self.websockets[sess_id] + del self.websockets[user_id] except KeyError: - log.exception("Non-existent websocket") + log.exception("Non-existent websocket for %s" % user_id) if sess_id in self.out_channels: try: self.out_channels[sess_id].close() except ChannelClosed: log.exception("Pika client (out) channel already closed") - def create_out_channel(self, sess_id): + def create_out_channel(self, sess_id, user_id): def _on_output_channel_creation(channel): def _on_output_queue_decleration(queue): channel.basic_consume(self.on_message, queue=sess_id) - log.debug("BIND QUEUE TO WS Q.%s on Ch.%s" % (sess_id, channel.consumer_tags[0])) + log.debug("BIND QUEUE TO WS Q.%s on Ch.%s WS.%s" % (sess_id, + channel.consumer_tags[0], + user_id)) self.out_channels[sess_id] = channel channel.queue_declare(callback=_on_output_queue_decleration, @@ -243,10 +246,10 @@ def redirect_incoming_message(self, sess_id, message, request): body=json_encode(message)) def on_message(self, channel, method, header, body): - sess_id = method.routing_key - log.debug("WS RPLY for %s: %s" % (sess_id, body)) - if sess_id in self.websockets: - self.websockets[sess_id].write_message(body) + user_id = method.exchange[4:] + log.debug("WS RPLY for %s: %s" % (user_id, body)) + if user_id in self.websockets: + self.websockets[user_id].write_message(body) channel.basic_ack(delivery_tag=method.delivery_tag) # else: From bf870644f5157df18468a49be67cb7dd8db49e16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Thu, 30 Jun 2016 08:18:19 +0300 Subject: [PATCH 16/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/tornado_server/queue_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zengine/tornado_server/queue_manager.py b/zengine/tornado_server/queue_manager.py index c9465c16..76ab68d1 100644 --- a/zengine/tornado_server/queue_manager.py +++ b/zengine/tornado_server/queue_manager.py @@ -72,6 +72,7 @@ def _send_message(self, sess_id, input_data): body=json_encode(input_data)) def _store_user_id(self, sess_id, body): + log.debug("SET SESSUSERS: %s" % sys.sessid_to_userid) sys.sessid_to_userid[sess_id[5:]] = json_decode(body)['user_id'].lower() def _wait_for_reply(self, sess_id, input_data): @@ -193,6 +194,7 @@ def register_websocket(self, sess_id, ws): sess_id: ws: """ + log.debug("GET SESSUSERS: %s" % sys.sessid_to_userid) user_id = sys.sessid_to_userid[sess_id] self.websockets[user_id] = ws self.create_out_channel(sess_id, user_id) From f13faf946b969a06edfd20099426862e8dae5412 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Thu, 30 Jun 2016 21:07:55 +0300 Subject: [PATCH 17/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/client_queue.py | 31 +------------------ zengine/management_commands.py | 8 +++-- zengine/messaging/lib.py | 2 +- zengine/messaging/model.py | 2 +- zengine/settings.py | 1 + zengine/tornado_server/get_logger.py | 4 +-- zengine/tornado_server/server.py | 2 +- .../{queue_manager.py => ws_to_queue.py} | 23 ++++++++++++-- zengine/views/system.py | 15 +++++++++ 9 files changed, 48 insertions(+), 40 deletions(-) rename zengine/tornado_server/{queue_manager.py => ws_to_queue.py} (91%) create mode 100644 zengine/views/system.py diff --git a/zengine/client_queue.py b/zengine/client_queue.py index fbbd4d8a..c8a8197b 100644 --- a/zengine/client_queue.py +++ b/zengine/client_queue.py @@ -13,7 +13,7 @@ import time from pika.exceptions import ConnectionClosed, ChannelClosed -from zengine.lib.cache import UserSessionID + BLOCKING_MQ_PARAMS = pika.ConnectionParameters( host=settings.MQ_HOST, @@ -47,11 +47,6 @@ def get_channel(self): self.connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) self.channel = pika.BlockingConnection(BLOCKING_MQ_PARAMS) return self.channel - # - # def get_sess_id(self): - # if not self.sess_id: - # self.sess_id = UserSessionID(self.user_id).get() - # return self.sess_id def send_to_default_exchange(self, sess_id, message=None): msg = json.dumps(message) @@ -65,27 +60,3 @@ def send_to_prv_exchange(self, user_id, message=None): log.debug("Sending following users \"%s\" exchange:\n%s " % (exchange, msg)) self.get_channel().publish(exchange=exchange, routing_key='', body=msg) - # def old_to_new_queue(self, old_sess_id): - # """ - # Somehow if users old (obsolete) queue has - # undelivered messages, we should redirect them to - # current queue. - # """ - # old_input_channel = self.connection.channel() - # while True: - # try: - # method_frame, header_frame, body = old_input_channel.basic_get(old_sess_id) - # if method_frame: - # self.send_to_queue(json_message=body) - # old_input_channel.basic_ack(method_frame.delivery_tag) - # else: - # old_input_channel.queue_delete(old_sess_id) - # old_input_channel.close() - # break - # except ChannelClosed as e: - # if e[0] == 404: - # break - # # e => (404, "NOT_FOUND - no queue 'sess_id' in vhost '/'") - # else: - # raise - # # old_input_channel = self.connection.channel() diff --git a/zengine/management_commands.py b/zengine/management_commands.py index 78acd24b..746147e3 100644 --- a/zengine/management_commands.py +++ b/zengine/management_commands.py @@ -8,6 +8,7 @@ # (GPLv3). See LICENSE.txt for details. import six +from pyoko.db.adapter.db_riak import BlockSave from pyoko.exceptions import ObjectDoesNotExist from pyoko.lib.utils import get_object_from_path from pyoko.manage import * @@ -200,9 +201,10 @@ def run(self): def create_user_channels(self): from zengine.messaging.model import Channel user_model = get_object_from_path(settings.USER_MODEL) - for usr in user_model.objects.filter(): - ch, new = Channel.objects.get_or_create(owner=usr, is_private=True) - print("%s exchange: %s" % ('created' if new else 'existing', ch.code_name)) + with BlockSave(Channel): + for usr in user_model.objects.filter(): + ch, new = Channel.objects.get_or_create(owner=usr, is_private=True) + print("%s exchange: %s" % ('created' if new else 'existing', ch.code_name)) def create_channel_exchanges(self): from zengine.messaging.model import Channel diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py index 3f397b60..a74009ff 100644 --- a/zengine/messaging/lib.py +++ b/zengine/messaging/lib.py @@ -108,7 +108,7 @@ def full_name(self): @property def prv_exchange(self): - return 'prv_%s' % self.key.lower() + return 'prv_%s' % str(self.key).lower() def bind_private_channel(self, sess_id): mq_channel = pika.BlockingConnection(BLOCKING_MQ_PARAMS).channel() diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index b69c4cd9..cd3663b1 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -93,7 +93,7 @@ def get_or_create_direct_channel(cls, initiator, receiver): def add_message(self, channel_key, body, title=None, sender=None, url=None, typ=2, receiver=None): mq_channel = self._connect_mq() mq_msg = json.dumps(dict(sender=sender, body=body, msg_title=title, url=url, typ=typ)) - mq_channel.basic_publish(exchange=channel_key, body=mq_msg) + mq_channel.basic_publish(exchange=channel_key, routing_key='', body=mq_msg) return Message(sender=sender, body=body, msg_title=title, url=url, typ=typ, channel_id=channel_key, receiver=receiver).save() diff --git a/zengine/settings.py b/zengine/settings.py index c3d0cf0f..ba94d4f3 100644 --- a/zengine/settings.py +++ b/zengine/settings.py @@ -114,6 +114,7 @@ #: ('URI template', 'python path to view method/class'), VIEW_URLS = { 'dashboard': 'zengine.views.menu.Menu', + 'sessid_to_userid': 'zengine.views.system.sessid_to_userid', 'ping': 'zengine.views.dev_utils.Ping', '_zops_create_message': 'zengine.messaging.views.create_message', 'mark_offline_user': 'zengine.messaging.views.mark_offline_user', diff --git a/zengine/tornado_server/get_logger.py b/zengine/tornado_server/get_logger.py index 06e8cca8..a0a84036 100644 --- a/zengine/tornado_server/get_logger.py +++ b/zengine/tornado_server/get_logger.py @@ -24,8 +24,8 @@ def get_logger(settings): # create formatter if settings.DEBUG: - # make log messages concise and readble for developemnt - format_str = '%(created)d - %(filename)s:%(lineno)d [%(module)s > %(funcName)s] - %(name)s - %(levelname)s - %(message)s' + # make log messages more readable at development + format_str = '%(asctime)s - %(filename)s:%(lineno)d %(module)s.%(funcName)s \n> %(message)s\n\n' else: format_str = '%(asctime)s - %(process)d - %(pathname)s:%(lineno)d [%(module)s > %(funcName)s] - %(name)s - %(levelname)s - %(message)s' diff --git a/zengine/tornado_server/server.py b/zengine/tornado_server/server.py index 671d535f..7c13e8b7 100644 --- a/zengine/tornado_server/server.py +++ b/zengine/tornado_server/server.py @@ -15,7 +15,7 @@ from tornado.httpclient import HTTPError sys.path.insert(0, os.path.realpath(os.path.dirname(__file__))) -from queue_manager import QueueManager, BlockingConnectionForHTTP, log +from ws_to_queue import QueueManager, BlockingConnectionForHTTP, log COOKIE_NAME = 'zopsess' DEBUG = os.getenv("DEBUG", False) diff --git a/zengine/tornado_server/queue_manager.py b/zengine/tornado_server/ws_to_queue.py similarity index 91% rename from zengine/tornado_server/queue_manager.py rename to zengine/tornado_server/ws_to_queue.py index 76ab68d1..4c004af3 100644 --- a/zengine/tornado_server/queue_manager.py +++ b/zengine/tornado_server/ws_to_queue.py @@ -186,6 +186,13 @@ def on_input_queue_declare(self, queue): exchange='input_exc', queue=self.INPUT_QUEUE_NAME, routing_key="#") + def ask_for_user_id(self, sess_id): + log.debug(sess_id) + # TODO: add remote ip + self.publish_incoming_message({'view': 'sessid_to_userid', + '_zops_remote_ip': '', + }, sess_id) + def register_websocket(self, sess_id, ws): """ @@ -195,9 +202,14 @@ def register_websocket(self, sess_id, ws): ws: """ log.debug("GET SESSUSERS: %s" % sys.sessid_to_userid) - user_id = sys.sessid_to_userid[sess_id] + try: + user_id = sys.sessid_to_userid[sess_id] + except KeyError: + self.ask_for_user_id(sess_id) + self.websockets[sess_id] = ws self.websockets[user_id] = ws self.create_out_channel(sess_id, user_id) + return True def inform_disconnection(self, sess_id): self.in_channel.basic_publish(exchange='input_exc', @@ -243,6 +255,9 @@ def redirect_incoming_message(self, sess_id, message, request): message = json_decode(message) message['_zops_sess_id'] = sess_id message['_zops_remote_ip'] = request.remote_ip + self.publish_incoming_message(message, sess_id) + + def publish_incoming_message(self, message, sess_id): self.in_channel.basic_publish(exchange='input_exc', routing_key=sess_id, body=json_encode(message)) @@ -252,7 +267,11 @@ def on_message(self, channel, method, header, body): log.debug("WS RPLY for %s: %s" % (user_id, body)) if user_id in self.websockets: self.websockets[user_id].write_message(body) - + channel.basic_ack(delivery_tag=method.delivery_tag) + elif 'sessid_to_userid' in body: + reply = json_decode(body) + sys.sessid_to_userid[reply['sess_id']] = reply['user_id'] + self.websockets[reply['user_id']] = self.websockets[reply['sess_id']] channel.basic_ack(delivery_tag=method.delivery_tag) # else: # channel.basic_reject(delivery_tag=method.delivery_tag) diff --git a/zengine/views/system.py b/zengine/views/system.py new file mode 100644 index 00000000..8989045c --- /dev/null +++ b/zengine/views/system.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +""" +""" + +# Copyright (C) 2015 ZetaOps Inc. +# +# This file is licensed under the GNU General Public License v3 +# (GPLv3). See LICENSE.txt for details. + +from zengine.lib.cache import UserSessionID + +def sessid_to_userid(current): + current.output['user_id'] = current.user_id.lower() + current.output['sess_id'] = current.session.sess_id + current.output['sessid_to_userid'] = True From 2e5c6236f00424c584a3816a8a87bbdbbb360099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Thu, 30 Jun 2016 22:40:52 +0300 Subject: [PATCH 18/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/tornado_server/ws_to_queue.py | 11 ++++++----- zengine/views/system.py | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/zengine/tornado_server/ws_to_queue.py b/zengine/tornado_server/ws_to_queue.py index 4c004af3..146a3d5f 100644 --- a/zengine/tornado_server/ws_to_queue.py +++ b/zengine/tornado_server/ws_to_queue.py @@ -189,9 +189,8 @@ def on_input_queue_declare(self, queue): def ask_for_user_id(self, sess_id): log.debug(sess_id) # TODO: add remote ip - self.publish_incoming_message({'view': 'sessid_to_userid', - '_zops_remote_ip': '', - }, sess_id) + self.publish_incoming_message(dict(_zops_remote_ip='', + data={'view': 'sessid_to_userid'}), sess_id) def register_websocket(self, sess_id, ws): @@ -204,12 +203,12 @@ def register_websocket(self, sess_id, ws): log.debug("GET SESSUSERS: %s" % sys.sessid_to_userid) try: user_id = sys.sessid_to_userid[sess_id] + self.websockets[user_id] = ws except KeyError: self.ask_for_user_id(sess_id) self.websockets[sess_id] = ws - self.websockets[user_id] = ws + user_id = sess_id self.create_out_channel(sess_id, user_id) - return True def inform_disconnection(self, sess_id): self.in_channel.basic_publish(exchange='input_exc', @@ -272,6 +271,8 @@ def on_message(self, channel, method, header, body): reply = json_decode(body) sys.sessid_to_userid[reply['sess_id']] = reply['user_id'] self.websockets[reply['user_id']] = self.websockets[reply['sess_id']] + del self.websockets[reply['sess_id']] channel.basic_ack(delivery_tag=method.delivery_tag) + # else: # channel.basic_reject(delivery_tag=method.delivery_tag) diff --git a/zengine/views/system.py b/zengine/views/system.py index 8989045c..e2215f28 100644 --- a/zengine/views/system.py +++ b/zengine/views/system.py @@ -12,4 +12,5 @@ def sessid_to_userid(current): current.output['user_id'] = current.user_id.lower() current.output['sess_id'] = current.session.sess_id + current.user.bind_private_channel(current.session.sess_id) current.output['sessid_to_userid'] = True From 7b3448efad42244317fc8d21a232bb2c6c39f6d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Fri, 8 Jul 2016 12:13:54 +0300 Subject: [PATCH 19/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/management_commands.py | 13 ++++++-- zengine/messaging/lib.py | 9 ++++- zengine/messaging/model.py | 57 ++++++++++++++++++-------------- zengine/messaging/views.py | 60 ++++++++++++++++++++-------------- zengine/models/auth.py | 5 +++ zengine/settings.py | 5 +-- zengine/views/system.py | 4 ++- 7 files changed, 96 insertions(+), 57 deletions(-) diff --git a/zengine/management_commands.py b/zengine/management_commands.py index 746147e3..35e97b8e 100644 --- a/zengine/management_commands.py +++ b/zengine/management_commands.py @@ -192,19 +192,26 @@ class PrepareMQ(Command): Creates necessary exchanges, queues and bindings """ CMD_NAME = 'preparemq' - HELP = 'Creates necessary exchanges, queues and bindings' + HELP = 'Creates necessary exchanges, queues and bindings for messaging subsystem' def run(self): self.create_user_channels() self.create_channel_exchanges() def create_user_channels(self): - from zengine.messaging.model import Channel + from zengine.messaging.model import Channel, Subscriber user_model = get_object_from_path(settings.USER_MODEL) with BlockSave(Channel): for usr in user_model.objects.filter(): - ch, new = Channel.objects.get_or_create(owner=usr, is_private=True) + # create private exchange of user + ch, new = Channel.objects.get_or_create(owner=usr, typ=5) print("%s exchange: %s" % ('created' if new else 'existing', ch.code_name)) + # create notification subscription to private exchange + sb, new = Subscriber.objects.get_or_create(channel=ch, user=usr, is_visible=False, + can_leave=False, inform_me=False) + print("%s notify sub: %s" % ('created' if new else 'existing', ch.code_name)) + + def create_channel_exchanges(self): from zengine.messaging.model import Channel diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py index a74009ff..ee43348b 100644 --- a/zengine/messaging/lib.py +++ b/zengine/messaging/lib.py @@ -16,6 +16,7 @@ from zengine.lib.cache import Cache from zengine.log import log + class ConnectionStatus(Cache): """ Cache object for workflow instances. @@ -70,11 +71,17 @@ def is_online(self, status=None): pass # TODO: do - def pre_save(self): + def encrypt_password(self): """ encrypt password if not already encrypted """ if self.password and not self.password.startswith('$pbkdf2'): self.set_password(self.password) + def prepare_channels(self): + from zengine.messaging.model import Channel, Subscriber + ch, new = Channel.objects.get_or_create(owner=self, typ=5) + sb, new = Subscriber.objects.get_or_create(channel=ch, user=self, is_visible=False, + can_leave=False, inform_me=False) + def check_password(self, raw_password): """ Verilen encrypt edilmemiş şifreyle kullanıcıya ait encrypt diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index cd3663b1..a83299c7 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -26,12 +26,16 @@ def get_mq_connection(): return connection, channel -# CHANNEL_TYPES = ( -# (1, "Notification"), -# (10, "System Broadcast"), -# (20, "Chat"), -# (25, "Direct"), -# ) +CHANNEL_TYPES = ( + # users private message hub + (5, "Private"), + # system notifications of user + # (10, "Notify"), + # a One-To-One communication between 2 user + (10, "Direct"), + # public chat rooms + (15, "Public"), +) class Channel(Model): @@ -47,24 +51,15 @@ class Channel(Model): mq_channel = None mq_connection = None + typ = field.Integer("Type", choices=CHANNEL_TYPES) name = field.String("Name") code_name = field.String("Internal name") description = field.String("Description") - owner = UserModel(reverse_name='created_channels') - # is this users private exchange - is_private = field.Boolean() - # is this a One-To-One channel - is_direct = field.Boolean() - - # typ = field.Integer("Type", choices=CHANNEL_TYPES) - - class Meta: - unique_together = (('is_private', 'owner'),) + owner = UserModel(reverse_name='created_channels', null=True) class Managers(ListNode): user = UserModel() - @classmethod def get_or_create_direct_channel(cls, initiator, receiver): """ @@ -90,8 +85,9 @@ def get_or_create_direct_channel(cls, initiator, receiver): return channel @classmethod - def add_message(self, channel_key, body, title=None, sender=None, url=None, typ=2, receiver=None): - mq_channel = self._connect_mq() + def add_message(cls, channel_key, body, title=None, sender=None, url=None, typ=2, + receiver=None): + mq_channel = cls._connect_mq() mq_msg = json.dumps(dict(sender=sender, body=body, msg_title=title, url=url, typ=typ)) mq_channel.basic_publish(exchange=channel_key, routing_key='', body=mq_msg) return Message(sender=sender, body=body, msg_title=title, url=url, @@ -117,6 +113,16 @@ def create_exchange(self): exchange_type='fanout', durable=True) + def get_actions_for(self, user): + actions = [ + ('Pin', 'pin_channel') + ] + if self.sender == user: + actions.extend([ + ('Delete', 'zops_delete_channel'), + ('Edit', 'zops_edit_channel') + ]) + def pre_creation(self): if not self.code_name: if self.name: @@ -142,11 +148,11 @@ class Subscriber(Model): channel = Channel() user = UserModel(reverse_name='subscriptions') - is_muted = field.Boolean("Mute the channel") + is_muted = field.Boolean("Mute the channel", default=False) inform_me = field.Boolean("Inform when I'm mentioned", default=True) - visible = field.Boolean("Show under user's channel list", default=True) + is_visible = field.Boolean("Show under user's channel list", default=True) can_leave = field.Boolean("Membership is not obligatory", default=True) - last_seen = field.DateTime("Last seen time") + last_seen_msg_time = field.DateTime("Last seen message's time") # status = field.Integer("Status", choices=SUBSCRIPTION_STATUS) @@ -158,7 +164,8 @@ def _connect_mq(cls): def unread_count(self): # FIXME: track and return actual unread message count - return 0 + return self.channel.message_set.objects.filter( + timestamp__lt=self.last_seen_msg_time).count() def create_exchange(self): """ @@ -236,8 +243,8 @@ def get_actions_for(self, user): ] if self.sender == user: actions.extend([ - ('Delete', 'delete_message'), - ('Edit', 'delete_message') + ('Delete', 'zops_delete_message'), + ('Edit', 'zops_edit_message') ]) else: actions.extend([ diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 7d520304..6a3e1297 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -8,7 +8,7 @@ # (GPLv3). See LICENSE.txt for details. from pyoko.conf import settings from pyoko.lib.utils import get_object_from_path -from zengine.messaging.model import Channel, Attachment +from zengine.messaging.model import Channel, Attachment, Subscriber from zengine.views.base import BaseView UserModel = get_object_from_path(settings.USER_MODEL) @@ -67,7 +67,7 @@ def show_channel(current): # request: { - 'view':'_zops_show_public_channel', + 'view':'_zops_show_channel', 'channel_key': key, } @@ -108,7 +108,8 @@ def show_channel(current): for msg in ch.get_last_messages()] } -def last_seen_channel(current): + +def last_seen_msg(current): """ Initial display of channel content. Returns channel description, members, no of members, last 20 messages etc. @@ -124,40 +125,49 @@ def last_seen_channel(current): 'msg_date': datetime, } + # response: + None + """ + Subscriber.objects.filter(channel_id=current.input['channel_key'], + user_id=current.user_id + ).update(last_seen_msg_time=current.input['msg_date']) + + + + + +def list_channels(current): + """ + List channel memberships of current user + + + .. code-block:: python + + # request: + { + 'view':'_zops_list_channels', + } + # response: { - 'channel_key': key, - 'description': string, - 'no_of_members': int, - 'member_list': [ + 'channels': [ {'name': string, - 'is_online': bool, - 'avatar_url': string, - }], - 'last_messages': [ - {'content': string, - 'title': string, - 'channel_key': key, - 'sender_name': string, - 'sender_key': key, + 'key': key, + 'unread': int, 'type': int, 'key': key, 'actions':[('name_string', 'cmd_string'),] } ] } - """ - current.input['seen_channel'] - -def mark_offline_user(current): - current.user.is_online(False) - -def list_channels(current): - return [ + """ + current.output['channels'] = [ {'name': sbs.channel.name, 'key': sbs.channel.key, + 'type': sbs.channel.typ, + 'actions': sbs.channel.get_actions_for(current.user), 'unread': sbs.unread_count()} for sbs in - current.user.subscriptions] + current.user.subscriptions if sbs.is_visible] def create_public_channel(current): diff --git a/zengine/models/auth.py b/zengine/models/auth.py index deb3af65..986595da 100644 --- a/zengine/models/auth.py +++ b/zengine/models/auth.py @@ -56,6 +56,11 @@ class Meta: """ list_fields = ['username', 'superuser'] + def pre_save(self): + self.encrypt_password() + + def post_creation(self): + self.prepare_channels() def get_permissions(self): """ diff --git a/zengine/settings.py b/zengine/settings.py index ba94d4f3..b75f30d2 100644 --- a/zengine/settings.py +++ b/zengine/settings.py @@ -115,10 +115,11 @@ VIEW_URLS = { 'dashboard': 'zengine.views.menu.Menu', 'sessid_to_userid': 'zengine.views.system.sessid_to_userid', + 'mark_offline_user': 'zengine.views.system.mark_offline_user', 'ping': 'zengine.views.dev_utils.Ping', '_zops_create_message': 'zengine.messaging.views.create_message', - 'mark_offline_user': 'zengine.messaging.views.mark_offline_user', - 'show_channel': 'zengine.messaging.views.show_channel', + '_zops_show_channel': 'zengine.messaging.views.show_channel', + '_zops_list_channels': 'zengine.messaging.views.list_channels', } if DEBUG: diff --git a/zengine/views/system.py b/zengine/views/system.py index e2215f28..a592faf2 100644 --- a/zengine/views/system.py +++ b/zengine/views/system.py @@ -7,10 +7,12 @@ # This file is licensed under the GNU General Public License v3 # (GPLv3). See LICENSE.txt for details. -from zengine.lib.cache import UserSessionID def sessid_to_userid(current): current.output['user_id'] = current.user_id.lower() current.output['sess_id'] = current.session.sess_id current.user.bind_private_channel(current.session.sess_id) current.output['sessid_to_userid'] = True + +def mark_offline_user(current): + current.user.is_online(False) From cb95cdf9fe43f8ec1fbca200a9a13860dd7d28b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Sat, 9 Jul 2016 01:30:39 +0300 Subject: [PATCH 20/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/model.py | 2 +- zengine/messaging/permissions.py | 16 ++++ zengine/messaging/views.py | 128 ++++++++++++++++++------------- 3 files changed, 93 insertions(+), 53 deletions(-) create mode 100644 zengine/messaging/permissions.py diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index a83299c7..9fe3a1fa 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -94,7 +94,7 @@ def add_message(cls, channel_key, body, title=None, sender=None, url=None, typ=2 typ=typ, channel_id=channel_key, receiver=receiver).save() def get_last_messages(self): - # TODO: Refactor this with RabbitMQ Last Cached Messages exchange + # TODO: Try to refactor this with https://github.com/rabbitmq/rabbitmq-recent-history-exchange return self.message_set.objects.filter()[:20] @classmethod diff --git a/zengine/messaging/permissions.py b/zengine/messaging/permissions.py new file mode 100644 index 00000000..0e422d7e --- /dev/null +++ b/zengine/messaging/permissions.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +""" +""" + +# Copyright (C) 2015 ZetaOps Inc. +# +# This file is licensed under the GNU General Public License v3 +# (GPLv3). See LICENSE.txt for details. +from zengine.auth.permissions import CustomPermission + +CustomPermission.add_multi( + # ('code_name', 'human_readable_name', 'description'), + [ + ('messaging.can_invite_user_by_unit', 'Can invite all users of a unit', ''), + ('messaging.can_invite_user_by_searching', 'Can invite any user by searching on name', ''), + ]) diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 6a3e1297..79b3f8a7 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -24,20 +24,21 @@ def create_message(current): { 'view':'_zops_create_message', 'message': { - 'channel': key, # of channel", - 'receiver': key, " of receiver. Should be set only for direct messages", - 'title': "Title of the message. Can be blank.", - 'body': "Message body.", - 'type': zengine.messaging.model.MSG_TYPES, + 'channel': key, # of channel + 'receiver': key, # of receiver. Should be set only for direct messages, + 'body': string, # message text., + 'type': int, # zengine.messaging.model.MSG_TYPES, 'attachments': [{ - 'description': "Can be blank.", - 'name': "File name with extension.", - 'content': "base64 encoded file content" + 'description': string, # can be blank, + 'name': string, # file name with extension, + 'content': string, # base64 encoded file content }]} # response: - { - 'msg_key': key, # of the just created message object, - } + { + 'status': string, # 'OK' for success + 'code': int, # 201 for success + 'msg_key': key, # key of the message object, + } """ msg = current.input['message'] @@ -73,31 +74,29 @@ def show_channel(current): # response: { - 'channel_key': key, - 'description': string, - 'no_of_members': int, - 'member_list': [ - {'name': string, - 'is_online': bool, - 'avatar_url': string, - }], - 'last_messages': [ - {'content': string, - 'title': string, - 'time': datetime, - 'channel_key': key, - 'sender_name': string, - 'sender_key': key, - 'type': int, - 'key': key, - 'actions':[('name_string', 'cmd_string'),] - } - ] + 'channel_key': key, + 'description': string, + 'no_of_members': int, + 'member_list': [ + {'name': string, + 'is_online': bool, + 'avatar_url': string, + }], + 'last_messages': [ + {'content': string, + 'title': string, + 'time': datetime, + 'channel_key': key, + 'sender_name': string, + 'sender_key': key, + 'type': int, + 'key': key, + 'actions':[('name_string', 'cmd_string'),] + }] } """ - ch_key = current.input['channel_key'] - ch = Channel.objects.get(ch_key) - current.output = {'channel_key': ch_key, + ch = Channel.objects.get(current.input['channel_key']) + current.output = {'channel_key': current.input['channel_key'], 'description': ch.description, 'no_of_members': len(ch.subscriber_set), 'member_list': [{'name': sb.user.full_name, @@ -111,29 +110,28 @@ def show_channel(current): def last_seen_msg(current): """ - Initial display of channel content. - Returns channel description, members, no of members, last 20 messages etc. + Push timestamp of last seen message for a channel .. code-block:: python # request: { - 'view':'_zops_last_seen_msg', - 'channel_key': key, - 'msg_key': key, - 'msg_date': datetime, + 'view':'_zops_last_seen_msg', + 'channel_key': key, + 'msg_key': key, + 'timestamp': datetime, } # response: - None + { + 'status': 'OK', + 'code': 200, + } """ Subscriber.objects.filter(channel_id=current.input['channel_key'], user_id=current.user_id - ).update(last_seen_msg_time=current.input['msg_date']) - - - + ).update(last_seen_msg_time=current.input['timestamp']) def list_channels(current): @@ -145,11 +143,11 @@ def list_channels(current): # request: { - 'view':'_zops_list_channels', + 'view':'_zops_list_channels', } - # response: - { + # response: + { 'channels': [ {'name': string, 'key': key, @@ -157,9 +155,8 @@ def list_channels(current): 'type': int, 'key': key, 'actions':[('name_string', 'cmd_string'),] - } - ] - } + },] + } """ current.output['channels'] = [ {'name': sbs.channel.name, @@ -171,7 +168,34 @@ def list_channels(current): def create_public_channel(current): - pass + """ + Create a public chat room + + .. code-block:: python + + # request: + { + 'view':'_zops_create_public_channel, + 'name': string, + 'description': string, + } + + # response: + { + 'status': 'Created', + 'code': 201, + 'channel_key': key, # of just created channel + } + """ + channel = Channel(name=current.input['name'], + description=current.input['description'], + owner=current.user, + typ=15).save() + current.output = { + 'channel_key': channel.key, + 'status': 'OK', + 'code': 201 + } def create_direct_channel(current): From b50c48cab05ed560bfff6afd77a6605973543424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Sat, 9 Jul 2016 20:16:46 +0300 Subject: [PATCH 21/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/views.py | 68 ++++++++++++++++++++++++++++++++++++++ zengine/settings.py | 5 +++ 2 files changed, 73 insertions(+) diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 79b3f8a7..9e392641 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -198,6 +198,74 @@ def create_public_channel(current): } +def add_members(current): + """ + Add member(s) to a public chat room + + .. code-block:: python + + # request: + { + 'view':'_zops_add_members, + 'channel_key': key, + 'members': [key, key], + } + + # response: + { + 'existing': [key,], # existing members + 'newly_added': [key,], # newly added members + 'status': 'Created', + 'code': 201 + } + """ + newly_added, existing = [], [] + for member_key in current.input['members']: + sb, new = Subscriber.objects.get_or_create(user_id=member_key, + channel_id=current.input['channel_key']) + if new: + newly_added.append(member_key) + else: + existing.append(member_key) + + current.output = { + 'existing': existing, + 'newly_added': newly_added, + 'status': 'OK', + 'code': 201 + } + +def search_user(current): + """ + Search users for adding to a public rooms + or creating one to one direct messaging + + .. code-block:: python + + # request: + { + 'view':'_zops_search_user, + 'query': string, + } + + # response: + { + 'results': [('full_name', 'key', 'avatar_url'), ], + 'status': 'OK', + 'code': 200 + } + """ + current.output = { + 'results': [], + 'status': 'OK', + 'code': 201 + } + for user in UserModel.objects.search_on(settings.MESSAGING_USER_SEARCH_FIELDS, + contains=current.input['query']): + current.input['results'].append((user.full_name, user.key, user.avatar)) + + + def create_direct_channel(current): """ Create a One-To-One channel for current user and selected user. diff --git a/zengine/settings.py b/zengine/settings.py index b75f30d2..b7c1cbbd 100644 --- a/zengine/settings.py +++ b/zengine/settings.py @@ -193,3 +193,8 @@ #: These models will not flushed when running tests TEST_FLUSHING_EXCLUDES = 'Permission,User,Role' + + +#: User search method of messaging subsystem will work on these fields +MESSAGING_USER_SEARCH_FIELDS = ['username', 'name', 'surname'] + From 408cd3739ad456881f788350953851045de603c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Sun, 10 Jul 2016 17:38:49 +0300 Subject: [PATCH 22/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/model.py | 21 +++---- zengine/messaging/views.py | 109 ++++++++++++++++++++++++++++++++----- zengine/models/auth.py | 28 +++++++++- zengine/settings.py | 6 ++ 4 files changed, 137 insertions(+), 27 deletions(-) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 9fe3a1fa..2bbbc7d6 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -56,32 +56,32 @@ class Channel(Model): code_name = field.String("Internal name") description = field.String("Description") owner = UserModel(reverse_name='created_channels', null=True) - - class Managers(ListNode): - user = UserModel() + # + # class Managers(ListNode): + # user = UserModel() @classmethod - def get_or_create_direct_channel(cls, initiator, receiver): + def get_or_create_direct_channel(cls, initiator_key, receiver_key): """ Creates a direct messaging channel between two user Args: - initiator: User, who sent the first message + initiator: User, who want's to make first contact receiver: User, other party Returns: Channel """ existing = cls.objects.OR().filter( - code_name='%s_%s' % (initiator.key, receiver.key)).filter( - code_name='%s_%s' % (receiver.key, initiator.key)) + code_name='%s_%s' % (initiator_key, receiver_key)).filter( + code_name='%s_%s' % (receiver_key, initiator_key)) if existing: return existing[0] else: - channel_name = '%s_%s' % (initiator.key, receiver.key) + channel_name = '%s_%s' % (initiator_key, receiver_key) channel = cls(is_direct=True, code_name=channel_name).save() - Subscriber(channel=channel, user=initiator).save() - Subscriber(channel=channel, user=receiver).save() + Subscriber(channel=channel, user_id=initiator_key).save() + Subscriber(channel=channel, user_id=receiver_key).save() return channel @classmethod @@ -151,6 +151,7 @@ class Subscriber(Model): is_muted = field.Boolean("Mute the channel", default=False) inform_me = field.Boolean("Inform when I'm mentioned", default=True) is_visible = field.Boolean("Show under user's channel list", default=True) + can_manage = field.Boolean("Can manage this channel", default=False) can_leave = field.Boolean("Membership is not obligatory", default=True) last_seen_msg_time = field.DateTime("Last seen message's time") diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 9e392641..dd93f10d 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -12,6 +12,7 @@ from zengine.views.base import BaseView UserModel = get_object_from_path(settings.USER_MODEL) +UnitModel = get_object_from_path(settings.UNIT_MODEL) def create_message(current): @@ -42,7 +43,7 @@ def create_message(current): """ msg = current.input['message'] - ch = Channel.objects.get(msg['channel']) + ch = Channel(current).objects.get(msg['channel']) msg_obj = ch.add_message(body=msg['body'], typ=msg['typ'], sender=current.user, title=msg['title'], receiver=msg['receiver'] or None) if 'attachment' in msg: @@ -95,7 +96,7 @@ def show_channel(current): }] } """ - ch = Channel.objects.get(current.input['channel_key']) + ch = Channel(current).objects.get(current.input['channel_key']) current.output = {'channel_key': current.input['channel_key'], 'description': ch.description, 'no_of_members': len(ch.subscriber_set), @@ -129,7 +130,7 @@ def last_seen_msg(current): 'code': 200, } """ - Subscriber.objects.filter(channel_id=current.input['channel_key'], + Subscriber(current).objects.filter(channel_id=current.input['channel_key'], user_id=current.user_id ).update(last_seen_msg_time=current.input['timestamp']) @@ -167,9 +168,9 @@ def list_channels(current): current.user.subscriptions if sbs.is_visible] -def create_public_channel(current): +def create_channel(current): """ - Create a public chat room + Create a public channel. Can be a broadcast channel or normal chat room. .. code-block:: python @@ -200,7 +201,7 @@ def create_public_channel(current): def add_members(current): """ - Add member(s) to a public chat room + Add member(s) to a channel .. code-block:: python @@ -221,7 +222,44 @@ def add_members(current): """ newly_added, existing = [], [] for member_key in current.input['members']: - sb, new = Subscriber.objects.get_or_create(user_id=member_key, + sb, new = Subscriber(current).objects.get_or_create(user_id=member_key, + channel_id=current.input['channel_key']) + if new: + newly_added.append(member_key) + else: + existing.append(member_key) + + current.output = { + 'existing': existing, + 'newly_added': newly_added, + 'status': 'OK', + 'code': 201 + } + + +def add_unit_to_channel(current): + """ + Subscribe users of a given unit to a channel + + .. code-block:: python + + # request: + { + 'view':'_zops_add_unit_to_channel, + 'unit_key': key, + } + + # response: + { + 'existing': [key,], # existing members + 'newly_added': [key,], # newly added members + 'status': 'Created', + 'code': 201 + } + """ + newly_added, existing = [], [] + for member_key in current.input['members']: + sb, new = Subscriber(current).objects.get_or_create(user_id=member_key, channel_id=current.input['channel_key']) if new: newly_added.append(member_key) @@ -260,26 +298,67 @@ def search_user(current): 'status': 'OK', 'code': 201 } - for user in UserModel.objects.search_on(settings.MESSAGING_USER_SEARCH_FIELDS, + for user in UserModel(current).objects.search_on(settings.MESSAGING_USER_SEARCH_FIELDS, contains=current.input['query']): current.input['results'].append((user.full_name, user.key, user.avatar)) +def search_unit(current): + """ + Search units for subscribing it's users to a channel + .. code-block:: python -def create_direct_channel(current): - """ - Create a One-To-One channel for current user and selected user. + # request: + { + 'view':'_zops_search_unit, + 'query': string, + } + # response: + { + 'results': [('name', 'key'), ], + 'status': 'OK', + 'code': 200 + } """ - pass + current.output = { + 'results': [], + 'status': 'OK', + 'code': 201 + } + for user in UnitModel(current).objects.search_on(settings.MESSAGING_UNIT_SEARCH_FIELDS, + contains=current.input['query']): + current.input['results'].append((user.name, user.key)) + + -def create_broadcast_channel(current): +def create_direct_channel(current): """ - Create a One-To-One channel for current user and selected user. + Create a One-To-One channel between current and selected user. + + .. code-block:: python + + # request: + { + 'view':'_zops_create_direct_channel, + 'user_key': key, + } + + # response: + { + 'status': 'Created', + 'code': 201, + 'channel_key': key, # of just created channel + } """ - pass + channel = Channel.get_or_create_direct_channel(current.user_id, current.input['user_key']) + current.output = { + 'channel_key': channel.key, + 'status': 'OK', + 'code': 201 + } def find_message(current): diff --git a/zengine/models/auth.py b/zengine/models/auth.py index 986595da..4c83051c 100644 --- a/zengine/models/auth.py +++ b/zengine/models/auth.py @@ -7,11 +7,35 @@ # This file is licensed under the GNU General Public License v3 # (GPLv3). See LICENSE.txt for details. -from pyoko import Model, field, ListNode +from pyoko import Model, field, ListNode, LinkProxy from passlib.hash import pbkdf2_sha512 from zengine.messaging.lib import BaseUser +class Unit(Model): + """Unit model + + Can be used do group users according to their physical or organizational position + + """ + name = field.String("İsim", index=True) + parent = LinkProxy('Unit', verbose_name='Parent Unit', reverse_name='sub_units') + + class Meta: + verbose_name = "Unit" + verbose_name_plural = "Units" + search_fields = ['name'] + list_fields = ['name',] + + def __unicode__(self): + return '%s' % self.name + + @classmethod + def get_user_keys(cls, current, unit_key): + return User(current).objects.filter(unit_id=unit_key).values_list('key', flatten=True) + + + class Permission(Model): """ Permission model @@ -50,6 +74,7 @@ class User(Model, BaseUser): password = field.String("Password") superuser = field.Boolean("Super user", default=False) avatar = field.File("Avatar", random_name=True, required=False) + unit = Unit() class Meta: """ meta class @@ -72,7 +97,6 @@ def get_permissions(self): users_primary_role = self.role_set[0].role return users_primary_role.get_permissions() - class Role(Model): """ This model binds group of Permissions with a certain User. diff --git a/zengine/settings.py b/zengine/settings.py index b7c1cbbd..9d8d7547 100644 --- a/zengine/settings.py +++ b/zengine/settings.py @@ -36,6 +36,9 @@ #: Role model ROLE_MODEL = 'zengine.models.Role' +#: Unit model +UNIT_MODEL = 'zengine.models.Unit' + MQ_HOST = os.getenv('MQ_HOST', 'localhost') MQ_PORT = int(os.getenv('MQ_PORT', '5672')) MQ_USER = os.getenv('MQ_USER', 'guest') @@ -198,3 +201,6 @@ #: User search method of messaging subsystem will work on these fields MESSAGING_USER_SEARCH_FIELDS = ['username', 'name', 'surname'] +#: Unit search method of messaging subsystem will work on these fields +MESSAGING_UNIT_SEARCH_FIELDS = ['name',] + From 1f70d8ae2016fe7503236bb93e2fbbf3962eb236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Mon, 11 Jul 2016 00:52:05 +0300 Subject: [PATCH 23/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/management_commands.py | 2 +- zengine/messaging/lib.py | 4 +- zengine/messaging/model.py | 3 ++ zengine/messaging/views.py | 86 +++++++++++++++++++++++++--------- 4 files changed, 70 insertions(+), 25 deletions(-) diff --git a/zengine/management_commands.py b/zengine/management_commands.py index 35e97b8e..86e2221c 100644 --- a/zengine/management_commands.py +++ b/zengine/management_commands.py @@ -207,7 +207,7 @@ def create_user_channels(self): ch, new = Channel.objects.get_or_create(owner=usr, typ=5) print("%s exchange: %s" % ('created' if new else 'existing', ch.code_name)) # create notification subscription to private exchange - sb, new = Subscriber.objects.get_or_create(channel=ch, user=usr, is_visible=False, + sb, new = Subscriber.objects.get_or_create(channel=ch, user=usr, is_visible=True, can_leave=False, inform_me=False) print("%s notify sub: %s" % ('created' if new else 'existing', ch.code_name)) diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py index ee43348b..94d4734d 100644 --- a/zengine/messaging/lib.py +++ b/zengine/messaging/lib.py @@ -78,8 +78,10 @@ def encrypt_password(self): def prepare_channels(self): from zengine.messaging.model import Channel, Subscriber + # create private channel of user ch, new = Channel.objects.get_or_create(owner=self, typ=5) - sb, new = Subscriber.objects.get_or_create(channel=ch, user=self, is_visible=False, + # create subscription entry for notification messages + sb, new = Subscriber.objects.get_or_create(channel=ch, user=self, is_visible=True, can_leave=False, inform_me=False) def check_password(self, raw_password): diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 2bbbc7d6..0ca977bc 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -60,6 +60,9 @@ class Channel(Model): # class Managers(ListNode): # user = UserModel() + def is_private(self): + return self.typ == 5 + @classmethod def get_or_create_direct_channel(cls, initiator_key, receiver_key): """ diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index dd93f10d..b4a93054 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -8,12 +8,33 @@ # (GPLv3). See LICENSE.txt for details. from pyoko.conf import settings from pyoko.lib.utils import get_object_from_path -from zengine.messaging.model import Channel, Attachment, Subscriber +from zengine.messaging.model import Channel, Attachment, Subscriber, Message from zengine.views.base import BaseView UserModel = get_object_from_path(settings.USER_MODEL) UnitModel = get_object_from_path(settings.UNIT_MODEL) +""" + +.. code-block:: python + + MSG_DICT_FORMAT = {'content': string, + 'title': string, + 'time': datetime, + 'channel_key': key, + 'sender_name': string, + 'sender_key': key, + 'type': int, + 'key': key, + 'actions':[('name_string', 'cmd_string'),] + 'attachments': [{ + 'description': string, + 'file_name': string, + 'url': string, + },] + } +""" + def create_message(current): """ @@ -83,17 +104,7 @@ def show_channel(current): 'is_online': bool, 'avatar_url': string, }], - 'last_messages': [ - {'content': string, - 'title': string, - 'time': datetime, - 'channel_key': key, - 'sender_name': string, - 'sender_key': key, - 'type': int, - 'key': key, - 'actions':[('name_string', 'cmd_string'),] - }] + 'last_messages': [MSG_DICT_FORMAT] } """ ch = Channel(current).objects.get(current.input['channel_key']) @@ -131,8 +142,8 @@ def last_seen_msg(current): } """ Subscriber(current).objects.filter(channel_id=current.input['channel_key'], - user_id=current.user_id - ).update(last_seen_msg_time=current.input['timestamp']) + user_id=current.user_id + ).update(last_seen_msg_time=current.input['timestamp']) def list_channels(current): @@ -160,7 +171,7 @@ def list_channels(current): } """ current.output['channels'] = [ - {'name': sbs.channel.name, + {'name': sbs.channel.name or ('Notifications' if sbs.channel.is_private() else ''), 'key': sbs.channel.key, 'type': sbs.channel.typ, 'actions': sbs.channel.get_actions_for(current.user), @@ -223,7 +234,7 @@ def add_members(current): newly_added, existing = [], [] for member_key in current.input['members']: sb, new = Subscriber(current).objects.get_or_create(user_id=member_key, - channel_id=current.input['channel_key']) + channel_id=current.input['channel_key']) if new: newly_added.append(member_key) else: @@ -258,9 +269,9 @@ def add_unit_to_channel(current): } """ newly_added, existing = [], [] - for member_key in current.input['members']: + for member_key in UnitModel.get_user_keys(current, current.input['unit_key']): sb, new = Subscriber(current).objects.get_or_create(user_id=member_key, - channel_id=current.input['channel_key']) + channel_id=current.input['channel_key']) if new: newly_added.append(member_key) else: @@ -273,6 +284,7 @@ def add_unit_to_channel(current): 'code': 201 } + def search_user(current): """ Search users for adding to a public rooms @@ -299,9 +311,10 @@ def search_user(current): 'code': 201 } for user in UserModel(current).objects.search_on(settings.MESSAGING_USER_SEARCH_FIELDS, - contains=current.input['query']): + contains=current.input['query']): current.input['results'].append((user.full_name, user.key, user.avatar)) + def search_unit(current): """ Search units for subscribing it's users to a channel @@ -327,12 +340,10 @@ def search_unit(current): 'code': 201 } for user in UnitModel(current).objects.search_on(settings.MESSAGING_UNIT_SEARCH_FIELDS, - contains=current.input['query']): + contains=current.input['query']): current.input['results'].append((user.name, user.key)) - - def create_direct_channel(current): """ Create a One-To-One channel between current and selected user. @@ -362,7 +373,36 @@ def create_direct_channel(current): def find_message(current): - pass + """ + Search in messages. If "channel_key" given, search will be limited in that channel. + + .. code-block:: python + + # request: + { + 'view':'_zops_search_unit, + 'channel_key': key, + 'query': string, + } + + # response: + { + 'results': [MSG_DICT_FORMAT, ], + 'status': 'OK', + 'code': 200 + } + """ + current.output = { + 'results': [], + 'status': 'OK', + 'code': 201 + } + query_set = Message(current).objects.search_on(['msg_title', 'body', 'url'], + contains=current.input['query']) + if current.input['channel_key']: + query_set = query_set.filter(channel_id=current.input['channel_key']) + for msg in query_set: + current.input['results'].append(msg.serialize_for(current.user)) def delete_message(current): From 64aa59fc09253fb248a2821c385365c82704c4a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Tue, 12 Jul 2016 06:01:22 +0300 Subject: [PATCH 24/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/views.py | 120 ++++++++++++++++++++++++++++++++----- 1 file changed, 105 insertions(+), 15 deletions(-) diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index b4a93054..e2fddc8c 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -7,7 +7,9 @@ # This file is licensed under the GNU General Public License v3 # (GPLv3). See LICENSE.txt for details. from pyoko.conf import settings +from pyoko.exceptions import ObjectDoesNotExist from pyoko.lib.utils import get_object_from_path +from zengine.lib.exceptions import HTTPError from zengine.messaging.model import Channel, Attachment, Subscriber, Message from zengine.views.base import BaseView @@ -18,7 +20,7 @@ .. code-block:: python - MSG_DICT_FORMAT = {'content': string, + MSG_DICT = {'content': string, 'title': string, 'time': datetime, 'channel_key': key, @@ -104,7 +106,7 @@ def show_channel(current): 'is_online': bool, 'avatar_url': string, }], - 'last_messages': [MSG_DICT_FORMAT] + 'last_messages': [MSG_DICT] } """ ch = Channel(current).objects.get(current.input['channel_key']) @@ -187,7 +189,7 @@ def create_channel(current): # request: { - 'view':'_zops_create_public_channel, + 'view':'_zops_create_public_channel', 'name': string, 'description': string, } @@ -218,7 +220,7 @@ def add_members(current): # request: { - 'view':'_zops_add_members, + 'view':'_zops_add_members', 'channel_key': key, 'members': [key, key], } @@ -256,7 +258,7 @@ def add_unit_to_channel(current): # request: { - 'view':'_zops_add_unit_to_channel, + 'view':'_zops_add_unit_to_channel', 'unit_key': key, } @@ -294,7 +296,7 @@ def search_user(current): # request: { - 'view':'_zops_search_user, + 'view':'_zops_search_user', 'query': string, } @@ -312,7 +314,7 @@ def search_user(current): } for user in UserModel(current).objects.search_on(settings.MESSAGING_USER_SEARCH_FIELDS, contains=current.input['query']): - current.input['results'].append((user.full_name, user.key, user.avatar)) + current.output['results'].append((user.full_name, user.key, user.avatar)) def search_unit(current): @@ -323,7 +325,7 @@ def search_unit(current): # request: { - 'view':'_zops_search_unit, + 'view':'_zops_search_unit', 'query': string, } @@ -341,7 +343,7 @@ def search_unit(current): } for user in UnitModel(current).objects.search_on(settings.MESSAGING_UNIT_SEARCH_FIELDS, contains=current.input['query']): - current.input['results'].append((user.name, user.key)) + current.output['results'].append((user.name, user.key)) def create_direct_channel(current): @@ -353,7 +355,7 @@ def create_direct_channel(current): # request: { - 'view':'_zops_create_direct_channel, + 'view':'_zops_create_direct_channel', 'user_key': key, } @@ -372,9 +374,37 @@ def create_direct_channel(current): } +def _paginate(self, current_page, query_set, per_page=10): + """ + Handles pagination of object listings. + + Args: + current_page int: + Current page number + query_set (:class:`QuerySet`): + Object listing queryset. + per_page int: + Objects per page. + + Returns: + QuerySet object, pagination data dict as a tuple + """ + total_objects = query_set.count() + total_pages = int(total_objects / per_page or 1) + # add orphans to last page + current_per_page = per_page + ( + total_objects % per_page if current_page == total_pages else 0) + pagination_data = dict(page=current_page, + total_pages=total_pages, + total_objects=total_objects, + per_page=current_per_page) + query_set = query_set.set_params(rows=current_per_page, start=(current_page - 1) * per_page) + return query_set, pagination_data + def find_message(current): """ - Search in messages. If "channel_key" given, search will be limited in that channel. + Search in messages. If "channel_key" given, search will be limited to that channel, + otherwise search will be performed on all of user's subscribed channels. .. code-block:: python @@ -383,11 +413,18 @@ def find_message(current): 'view':'_zops_search_unit, 'channel_key': key, 'query': string, + 'page': int, } # response: { - 'results': [MSG_DICT_FORMAT, ], + 'results': [MSG_DICT, ], + 'pagination': { + 'page': int, # current page + 'total_pages': int, + 'total_objects': int, + 'per_page': int, # object per page + }, 'status': 'OK', 'code': 200 } @@ -401,13 +438,66 @@ def find_message(current): contains=current.input['query']) if current.input['channel_key']: query_set = query_set.filter(channel_id=current.input['channel_key']) + else: + subscribed_channels = Subscriber.objects.filter(user_id=current.user_id).values_list( + "channel_id", flatten=True) + query_set = query_set.filter(channel_id__in=subscribed_channels) + + query_set, pagination_data = _paginate(current_page=current.input['page'], query_set=query_set) + current.output['pagination'] = pagination_data for msg in query_set: - current.input['results'].append(msg.serialize_for(current.user)) + current.output['results'].append(msg.serialize_for(current.user)) def delete_message(current): - pass + """ + Delete a message + + .. code-block:: python + # request: + { + 'view':'_zops_delete_message, + 'message_key': key, + } + + # response: + { + 'status': 'OK', + 'code': 200 + } + """ + try: + Message(current).objects.get(sender_id=current.user_id, key=current.input['message_key']).delete() + current.output = { + 'status': 'OK', + 'code': 201 + } + except ObjectDoesNotExist: + raise HTTPError(404, "") def edit_message(current): - pass + """ + Edit a message a user own. + + .. code-block:: python + + # request: + { + 'view':'_zops_edit_message', + 'message': { + 'body': string, # message text + 'key': key + } + } + # response: + { + 'status': string, # 'OK' for success + 'code': int, # 201 for success + } + + """ + msg = current.input['message'] + if not Message(current).objects.filter(sender_id=current.user_id, + key=msg['key']).update(body=msg['body']): + raise HTTPError(404, "") From f7f851daa50da9232508d73c90f98fa64adcee0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Tue, 12 Jul 2016 10:37:55 +0300 Subject: [PATCH 25/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/model.py | 27 ++++-- zengine/messaging/views.py | 186 ++++++++++++++++++++++++++++++------- zengine/readthedocs.yml | 5 + 3 files changed, 178 insertions(+), 40 deletions(-) create mode 100644 zengine/readthedocs.yml diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 0ca977bc..e52b761c 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -7,6 +7,7 @@ # This file is licensed under the GNU General Public License v3 # (GPLv3). See LICENSE.txt for details. import json +from uuid import uuid4 import pika @@ -56,6 +57,7 @@ class Channel(Model): code_name = field.String("Internal name") description = field.String("Description") owner = UserModel(reverse_name='created_channels', null=True) + # # class Managers(ListNode): # user = UserModel() @@ -88,13 +90,14 @@ def get_or_create_direct_channel(cls, initiator_key, receiver_key): return channel @classmethod - def add_message(cls, channel_key, body, title=None, sender=None, url=None, typ=2, - receiver=None): + def add_message(cls, channel_key, body, title=None, sender=None, url=None, typ=2, receiver=None): mq_channel = cls._connect_mq() - mq_msg = json.dumps(dict(sender=sender, body=body, msg_title=title, url=url, typ=typ)) - mq_channel.basic_publish(exchange=channel_key, routing_key='', body=mq_msg) - return Message(sender=sender, body=body, msg_title=title, url=url, - typ=typ, channel_id=channel_key, receiver=receiver).save() + msg_object = Message(sender=sender, body=body, msg_title=title, url=url, + typ=typ, channel_id=channel_key, receiver=receiver, key=uuid4().hex) + mq_channel.basic_publish(exchange=channel_key, + routing_key='', + body=json.dumps(msg_object.serialize_for())) + return msg_object.save() def get_last_messages(self): # TODO: Try to refactor this with https://github.com/rabbitmq/rabbitmq-recent-history-exchange @@ -250,12 +253,12 @@ def get_actions_for(self, user): ('Delete', 'zops_delete_message'), ('Edit', 'zops_edit_message') ]) - else: + elif user: actions.extend([ ('Flag', 'flag_message') ]) - def serialize_for(self, user): + def serialize_for(self, user=None): return { 'content': self.body, 'type': self.typ, @@ -311,3 +314,11 @@ class Favorite(Model): channel = Channel() user = UserModel() message = Message() + summary = field.String("Message Summary") + channel_name = field.String("Channel Name") + + def pre_creation(self): + if not self.channel: + self.channel = self.message.channel + self.summary = self.message.body[:60] + self.channel_name = self.channel.name diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index e2fddc8c..0584898c 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -10,8 +10,7 @@ from pyoko.exceptions import ObjectDoesNotExist from pyoko.lib.utils import get_object_from_path from zengine.lib.exceptions import HTTPError -from zengine.messaging.model import Channel, Attachment, Subscriber, Message -from zengine.views.base import BaseView +from zengine.messaging.model import Channel, Attachment, Subscriber, Message, Favorite UserModel = get_object_from_path(settings.USER_MODEL) UnitModel = get_object_from_path(settings.UNIT_MODEL) @@ -122,6 +121,35 @@ def show_channel(current): } +def channel_history(current): + """ + Get old messages for a channel. 20 messages per request + + .. code-block:: python + + # request: + { + 'view':'_zops_channel_history, + 'channel_key': key, + 'timestamp': datetime, # timestamp data of oldest shown message + } + + # response: + { + 'messages': [MSG_DICT, ], + 'status': 'OK', + 'code': 200 + } + """ + current.output = { + 'status': 'OK', + 'code': 201, + 'messages': [msg.serialize_for(current.user) + for msg in Message.objects.filter(channel_id=current.input['channel_key'], + timestamp__lt=current.input['timestamp'])[:20]] + } + + def last_seen_msg(current): """ Push timestamp of last seen message for a channel @@ -214,7 +242,7 @@ def create_channel(current): def add_members(current): """ - Add member(s) to a channel + Subscribe member(s) to a channel .. code-block:: python @@ -252,23 +280,25 @@ def add_members(current): def add_unit_to_channel(current): """ - Subscribe users of a given unit to a channel - - .. code-block:: python - - # request: - { - 'view':'_zops_add_unit_to_channel', - 'unit_key': key, - } - - # response: - { - 'existing': [key,], # existing members - 'newly_added': [key,], # newly added members - 'status': 'Created', - 'code': 201 - } + Subscribe users of a given unit to given channel + + JSON API: + .. code-block:: python + + # request: + { + 'view':'_zops_add_unit_to_channel', + 'unit_key': key, + 'channel_key': key, + } + + # response: + { + 'existing': [key,], # existing members + 'newly_added': [key,], # newly added members + 'status': 'Created', + 'code': 201 + } """ newly_added, existing = [], [] for member_key in UnitModel.get_user_keys(current, current.input['unit_key']): @@ -289,7 +319,7 @@ def add_unit_to_channel(current): def search_user(current): """ - Search users for adding to a public rooms + Search users for adding to a public room or creating one to one direct messaging .. code-block:: python @@ -319,7 +349,7 @@ def search_user(current): def search_unit(current): """ - Search units for subscribing it's users to a channel + Search on units for subscribing it's users to a channel .. code-block:: python @@ -395,12 +425,13 @@ def _paginate(self, current_page, query_set, per_page=10): current_per_page = per_page + ( total_objects % per_page if current_page == total_pages else 0) pagination_data = dict(page=current_page, - total_pages=total_pages, - total_objects=total_objects, - per_page=current_per_page) + total_pages=total_pages, + total_objects=total_objects, + per_page=current_per_page) query_set = query_set.set_params(rows=current_per_page, start=(current_page - 1) * per_page) return query_set, pagination_data + def find_message(current): """ Search in messages. If "channel_key" given, search will be limited to that channel, @@ -468,14 +499,13 @@ def delete_message(current): } """ try: - Message(current).objects.get(sender_id=current.user_id, key=current.input['message_key']).delete() - current.output = { - 'status': 'OK', - 'code': 201 - } + Message(current).objects.get(sender_id=current.user_id, + key=current.input['message_key']).delete() + current.output = {'status': 'Deleted', 'code': 200} except ObjectDoesNotExist: raise HTTPError(404, "") + def edit_message(current): """ Edit a message a user own. @@ -493,11 +523,103 @@ def edit_message(current): # response: { 'status': string, # 'OK' for success - 'code': int, # 201 for success + 'code': int, # 200 for success } """ + current.output = {'status': 'OK', 'code': 200} msg = current.input['message'] if not Message(current).objects.filter(sender_id=current.user_id, - key=msg['key']).update(body=msg['body']): + key=msg['key']).update(body=msg['body']): + raise HTTPError(404, "") + + +def add_to_favorites(current): + """ + Favorite a message + + .. code-block:: python + + # request: + { + 'view':'_zops_add_to_favorites, + 'message_key': key, + } + + # response: + { + 'status': 'Created', + 'code': 201 + 'favorite_key': key + } + + """ + msg = Message.objects.get(current.input['message_key']) + current.output = {'status': 'Created', 'code': 201} + fav, new = Favorite.objects.get_or_create(user_id=current.user_id, message=msg['key']) + current.output['favorite_key'] = fav.key + + +def remove_from_favorites(current): + """ + Remove a message from favorites + + .. code-block:: python + + # request: + { + 'view':'_zops_remove_from_favorites, + 'message_key': key, + } + + # response: + { + 'status': 'Deleted', + 'code': 200 + } + + """ + try: + current.output = {'status': 'Deleted', 'code': 200} + Favorite(current).objects.get(user_id=current.user_id, + key=current.input['message_key']).delete() + except ObjectDoesNotExist: raise HTTPError(404, "") + + +def list_favorites(current): + """ + List user's favorites. If "channel_key" given, will return favorites belong to that channel. + + .. code-block:: python + + # request: + { + 'view':'_zops_list_favorites, + 'channel_key': key, + } + + # response: + { + 'status': 'OK', + 'code': 200 + 'favorites':[{'key': key, + 'channel_key': key, + 'message_key': key, + 'message_summary': string, # max 60 char + 'channel_name': string, + },] + } + + """ + current.output = {'status': 'OK', 'code': 200, 'favorites': []} + query_set = Favorite(current).objects.filter(user_id=current.user_id) + if current.input['channel_key']: + query_set = query_set.filter(channel_id=current.input['channel_key']) + current.output['favorites'] = [{ + 'key': fav.key, + 'channel_key': fav.channel.key, + 'message_key': fav.message.key, + 'message_summary': fav.summary, + 'channel_name': fav.channel_name + } for fav in query_set] diff --git a/zengine/readthedocs.yml b/zengine/readthedocs.yml new file mode 100644 index 00000000..5568dc78 --- /dev/null +++ b/zengine/readthedocs.yml @@ -0,0 +1,5 @@ +requirements_file: requirements/default.txt + +python: + version: 2 + pip_install: true From 403de71732259bf56817edd590bdb81476348059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Tue, 12 Jul 2016 13:45:36 +0300 Subject: [PATCH 26/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/model.py | 2 - zengine/messaging/views.py | 95 ++++++++++++++++++++++------------ zengine/models/auth.py | 2 +- zengine/settings.py | 16 ++++++ zengine/views/catalog_datas.py | 5 +- 5 files changed, 81 insertions(+), 39 deletions(-) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index e52b761c..3c0037b7 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -30,8 +30,6 @@ def get_mq_connection(): CHANNEL_TYPES = ( # users private message hub (5, "Private"), - # system notifications of user - # (10, "Notify"), # a One-To-One communication between 2 user (10, "Direct"), # public chat rooms diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 0584898c..eb8d182a 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -36,6 +36,38 @@ } """ +def _dedect_file_type(name, content): + # FIXME: Implement attachment type detection + return 1 # Return as Document for now + + +def _paginate(self, current_page, query_set, per_page=10): + """ + Handles pagination of object listings. + + Args: + current_page int: + Current page number + query_set (:class:`QuerySet`): + Object listing queryset. + per_page int: + Objects per page. + + Returns: + QuerySet object, pagination data dict as a tuple + """ + total_objects = query_set.count() + total_pages = int(total_objects / per_page or 1) + # add orphans to last page + current_per_page = per_page + ( + total_objects % per_page if current_page == total_pages else 0) + pagination_data = dict(page=current_page, + total_pages=total_pages, + total_objects=total_objects, + per_page=current_per_page) + query_set = query_set.set_params(rows=current_per_page, start=(current_page - 1) * per_page) + return query_set, pagination_data + def create_message(current): """ @@ -76,10 +108,6 @@ def create_message(current): description=atch['description'], typ=typ).save() -def _dedect_file_type(name, content): - # FIXME: Implement attachment type detection - return 1 # Return as Document for now - def show_channel(current): """ @@ -144,9 +172,10 @@ def channel_history(current): current.output = { 'status': 'OK', 'code': 201, - 'messages': [msg.serialize_for(current.user) - for msg in Message.objects.filter(channel_id=current.input['channel_key'], - timestamp__lt=current.input['timestamp'])[:20]] + 'messages': [ + msg.serialize_for(current.user) + for msg in Message.objects.filter(channel_id=current.input['channel_key'], + timestamp__lt=current.input['timestamp'])[:20]] } @@ -404,33 +433,6 @@ def create_direct_channel(current): } -def _paginate(self, current_page, query_set, per_page=10): - """ - Handles pagination of object listings. - - Args: - current_page int: - Current page number - query_set (:class:`QuerySet`): - Object listing queryset. - per_page int: - Objects per page. - - Returns: - QuerySet object, pagination data dict as a tuple - """ - total_objects = query_set.count() - total_pages = int(total_objects / per_page or 1) - # add orphans to last page - current_per_page = per_page + ( - total_objects % per_page if current_page == total_pages else 0) - pagination_data = dict(page=current_page, - total_pages=total_pages, - total_objects=total_objects, - per_page=current_per_page) - query_set = query_set.set_params(rows=current_per_page, start=(current_page - 1) * per_page) - return query_set, pagination_data - def find_message(current): """ @@ -534,6 +536,31 @@ def edit_message(current): raise HTTPError(404, "") +def get_message_actions(current): + """ + Returns applicable actions for current user for given message key + + .. code-block:: python + + # request: + { + 'view':'_zops_get_message_actions', + 'message_key': key, + } + # response: + { + 'actions':[('name_string', 'cmd_string'),] + 'status': string, # 'OK' for success + 'code': int, # 200 for success + } + + """ + current.output = {'status': 'OK', + 'code': 200, + 'actions': Message.objects.get( + current.input['message_key']).get_actions_for(current.user)} + + def add_to_favorites(current): """ Favorite a message diff --git a/zengine/models/auth.py b/zengine/models/auth.py index 4c83051c..650abaeb 100644 --- a/zengine/models/auth.py +++ b/zengine/models/auth.py @@ -15,7 +15,7 @@ class Unit(Model): """Unit model - Can be used do group users according to their physical or organizational position + Can be used to group users according to their physical or organizational position """ name = field.String("İsim", index=True) diff --git a/zengine/settings.py b/zengine/settings.py index 9d8d7547..e81ce809 100644 --- a/zengine/settings.py +++ b/zengine/settings.py @@ -123,6 +123,22 @@ '_zops_create_message': 'zengine.messaging.views.create_message', '_zops_show_channel': 'zengine.messaging.views.show_channel', '_zops_list_channels': 'zengine.messaging.views.list_channels', + '_zops_channel_history': 'zengine.messaging.views.channel_history', + '_zops_last_seen_msg': 'zengine.messaging.views.last_seen_msg', + '_zops_create_channel': 'zengine.messaging.views.create_channel', + '_zops_add_members': 'zengine.messaging.views.add_members', + '_zops_add_unit_to_channel': 'zengine.messaging.views.add_unit_to_channel', + '_zops_search_user': 'zengine.messaging.views.search_user', + '_zops_search_unit': 'zengine.messaging.views.search_unit', + '_zops_create_direct_channel': 'zengine.messaging.views.create_direct_channel', + '_zops_find_message': 'zengine.messaging.views.find_message', + '_zops_delete_message': 'zengine.messaging.views.delete_message', + '_zops_edit_message': 'zengine.messaging.views.edit_message', + '_zops_get_message_actions': 'zengine.messaging.views.get_message_actions', + '_zops_add_to_favorites': 'zengine.messaging.views.add_to_favorites', + '_zops_remove_from_favorites': 'zengine.messaging.views.remove_from_favorites', + '_zops_list_favorites': 'zengine.messaging.views.list_favorites', + # '_zops_': 'zengine.messaging.views.', } if DEBUG: diff --git a/zengine/views/catalog_datas.py b/zengine/views/catalog_datas.py index e5f9a45f..8d786c64 100644 --- a/zengine/views/catalog_datas.py +++ b/zengine/views/catalog_datas.py @@ -13,13 +13,14 @@ from zengine.views.crud import CrudView from zengine import forms from zengine.forms import fields -from falcon import HTTPBadRequest +from zengine.lib.exceptions import HTTPError class CatalogSelectForm(forms.JsonForm): """ Generates Form object for catalog select view. """ + class Meta: title = 'Choose Catalog Data' help_text = "Type and choose existing catalog data to edit. Or if you want to add one type the name of the catalog data you want to add." @@ -121,6 +122,6 @@ def save_catalog(self): self.output["notify"] = "catalog: %s successfully updated." % self.input[ "object_key"] except: - raise HTTPBadRequest("Form object could not be saved") + raise HTTPError(500, "Form object could not be saved") if self.input["cmd"] == 'cancel': self.output["notify"] = "catalog: %s canceled." % self.input["object_key"] From 6e936eabddff80e3f7c34744904dc57b8333c606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Tue, 12 Jul 2016 17:59:19 +0300 Subject: [PATCH 27/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/client_queue.py | 18 ++++++++++++++++++ zengine/messaging/model.py | 11 +++++++---- zengine/messaging/views.py | 6 +++++- zengine/wf_daemon.py | 2 +- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/zengine/client_queue.py b/zengine/client_queue.py index c8a8197b..5e2a5bc1 100644 --- a/zengine/client_queue.py +++ b/zengine/client_queue.py @@ -49,12 +49,30 @@ def get_channel(self): return self.channel def send_to_default_exchange(self, sess_id, message=None): + """ + Send messages through RabbitMQ's default exchange, + which will be delivered through routing_key (sess_id). + + This method only used for un-authenticated users, i.e. login process. + + Args: + sess_id string: Session id + message dict: Message object. + """ msg = json.dumps(message) log.debug("Sending following message to %s queue through default exchange:\n%s" % ( sess_id, msg)) self.get_channel().publish(exchange='', routing_key=sess_id, body=msg) def send_to_prv_exchange(self, user_id, message=None): + """ + Send messages through logged in users private exchange. + + Args: + user_id string: User key + message dict: Message object + + """ exchange = 'prv_%s' % user_id.lower() msg = json.dumps(message) log.debug("Sending following users \"%s\" exchange:\n%s " % (exchange, msg)) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 3c0037b7..c78c4124 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -150,6 +150,9 @@ class Subscriber(Model): Permission model """ + mq_channel = None + mq_connection = None + channel = Channel() user = UserModel(reverse_name='subscriptions') is_muted = field.Boolean("Mute the channel", default=False) @@ -163,9 +166,9 @@ class Subscriber(Model): @classmethod def _connect_mq(cls): - if cls.connection is None or cls.connection.is_closed: - cls.connection, cls.channel = get_mq_connection() - return cls.channel + if cls.mq_connection is None or cls.mq_connection.is_closed: + cls.mq_connection, cls.mq_channel = get_mq_connection() + return cls.mq_channel def unread_count(self): # FIXME: track and return actual unread message count @@ -196,7 +199,7 @@ def bind_to_channel(self): Automatically called at creation of subscription record. """ channel = self._connect_mq() - channel.exchange_bind(source=self.channel.code_name, destination=self.user.key) + channel.exchange_bind(source=self.channel.code_name, destination=self.user.prv_exchange) def post_creation(self): self.create_exchange() diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index eb8d182a..3850fe9f 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -246,7 +246,7 @@ def create_channel(current): # request: { - 'view':'_zops_create_public_channel', + 'view':'_zops_create_channel', 'name': string, 'description': string, } @@ -279,6 +279,8 @@ def add_members(current): { 'view':'_zops_add_members', 'channel_key': key, + 'read_only': True, # True if this is a Broadcast channel, + # False if it's a normal chat room 'members': [key, key], } @@ -291,8 +293,10 @@ def add_members(current): } """ newly_added, existing = [], [] + read_only = current.input['read_only'] for member_key in current.input['members']: sb, new = Subscriber(current).objects.get_or_create(user_id=member_key, + read_only=read_only, channel_id=current.input['channel_key']) if new: newly_added.append(member_key) diff --git a/zengine/wf_daemon.py b/zengine/wf_daemon.py index 215b1271..4649ac15 100755 --- a/zengine/wf_daemon.py +++ b/zengine/wf_daemon.py @@ -122,7 +122,7 @@ def _handle_workflow(self, session, data, headers): def handle_message(self, ch, method, properties, body): """ this is a pika.basic_consumer callback - handles client inputs, runs appropriate workflows + handles client inputs, runs appropriate workflows and views Args: ch: amqp channel From 47692398742a0bf97816d6a22c4d4a8b5d9cc627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Wed, 13 Jul 2016 14:18:01 +0300 Subject: [PATCH 28/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/lib/test_utils.py | 30 +++++++-- zengine/messaging/model.py | 23 ++++--- zengine/messaging/views.py | 126 +++++++++++++++++++++++++++++++++---- zengine/models/auth.py | 8 ++- zengine/settings.py | 3 + 5 files changed, 162 insertions(+), 28 deletions(-) diff --git a/zengine/lib/test_utils.py b/zengine/lib/test_utils.py index 8fa2299f..d9a6a858 100644 --- a/zengine/lib/test_utils.py +++ b/zengine/lib/test_utils.py @@ -12,6 +12,7 @@ from zengine.lib.cache import ClearCache from zengine.lib.exceptions import HTTPError from zengine.log import log +from zengine.tornado_server.ws_to_queue import BlockingConnectionForHTTP from zengine.wf_daemon import Worker from zengine.models import User @@ -60,7 +61,7 @@ def raw(self): pprint(self.content) -class TestClient(Worker): +class BaseTestClient(Worker): """ TestClient to simplify writing API tests for Zengine based apps. """ @@ -71,7 +72,7 @@ def __init__(self, path, *args, **kwargs): :param str path: Request uri """ - super(TestClient, self).__init__(*args, **kwargs) + super(BaseTestClient, self).__init__(*args, **kwargs) self.test_client_sessid = None self.response_wrapper = None self.set_path(path, None) @@ -93,7 +94,7 @@ def set_path(self, path, token=''): self.path = path self.token = token - def post(self, **data): + def _prepare_post(self, **data): """ by default data dict encoded as json and content type set as application/json @@ -119,13 +120,32 @@ def post(self, **data): post_data = {'data': data, '_zops_remote_ip': '127.0.0.1'} log.info("PostData : %s" % post_data) print("PostData : %s" % post_data) - post_data = json.dumps(post_data) + return post_data + + def post(self, **data): + post_data = json.dumps(self._prepare_post(data)) fake_method = type('FakeMethod', (object,), {'routing_key': self.sess_id}) self.handle_message(None, fake_method, None, post_data) # update client token from response self.token = self.response_wrapper.token return self.response_wrapper + +class AMQPTestClient(BaseTestClient): + def apost(self, **data): + """ + AMQP based post method + Args: + **data: Post data + + Returns: + + """ + post_data = self._prepare_post(data) + BlockingConnectionForHTTP().send_message(self.sess_id, post_data) + + +class TestClient(BaseTestClient): def send_output(self, output): self.response_wrapper = ResponseWrapper(output) @@ -168,7 +188,7 @@ def setup_method(self, method): sleep(2) else: print( - "\nREPORT:: Test case does not have a fixture file like %s" % fixture_guess) + "\nREPORT:: Test case does not have a fixture file like %s" % fixture_guess) else: print("\nREPORT:: Fixture loading disabled by user. (by --ignore=fixture)") diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index c78c4124..093afef1 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -117,15 +117,7 @@ def create_exchange(self): exchange_type='fanout', durable=True) - def get_actions_for(self, user): - actions = [ - ('Pin', 'pin_channel') - ] - if self.sender == user: - actions.extend([ - ('Delete', 'zops_delete_channel'), - ('Edit', 'zops_edit_channel') - ]) + def pre_creation(self): if not self.code_name: @@ -156,7 +148,10 @@ class Subscriber(Model): channel = Channel() user = UserModel(reverse_name='subscriptions') is_muted = field.Boolean("Mute the channel", default=False) + pinned = field.Boolean("Pin channel to top", default=False) inform_me = field.Boolean("Inform when I'm mentioned", default=True) + read_only = field.Boolean("This is a read-only subscription (to a broadcast channel)", + default=False) is_visible = field.Boolean("Show under user's channel list", default=True) can_manage = field.Boolean("Can manage this channel", default=False) can_leave = field.Boolean("Membership is not obligatory", default=True) @@ -170,6 +165,16 @@ def _connect_mq(cls): cls.mq_connection, cls.mq_channel = get_mq_connection() return cls.mq_channel + def get_actions(self): + actions = [ + ('Pin', 'pin_channel') + ] + if self.channel.owner == self.user: + actions.extend([ + ('Delete', '_zops_delete_channel'), + ('Edit', '_zops_edit_channel') + ]) + def unread_count(self): # FIXME: track and return actual unread message count return self.channel.message_set.objects.filter( diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 3850fe9f..c30aec52 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -27,7 +27,15 @@ 'sender_key': key, 'type': int, 'key': key, - 'actions':[('name_string', 'cmd_string'),] + 'actions':[('action name', 'view name'), + ('Add to Favorite', '_zops_add_to_favorites'), # applicable to everyone + + # Additional actions should be retrieved + # from "_zops_get_message_actions" view. + ('Edit', '_zops_edit_message'), + ('Delete', '_zops_delete_message'), + + ] 'attachments': [{ 'description': string, 'file_name': string, @@ -36,6 +44,7 @@ } """ + def _dedect_file_type(name, content): # FIXME: Implement attachment type detection return 1 # Return as Document for now @@ -108,7 +117,6 @@ def create_message(current): description=atch['description'], typ=typ).save() - def show_channel(current): """ Initial display of channel content. @@ -220,12 +228,19 @@ def list_channels(current): # response: { 'channels': [ - {'name': string, - 'key': key, - 'unread': int, - 'type': int, - 'key': key, - 'actions':[('name_string', 'cmd_string'),] + {'name': string, # name of channel + 'key': key, # key of channel + 'unread': int, # unread message count + 'type': int, # channel type, + # 15: public channels (chat room/broadcast channel distinction + comes from "read_only" flag) + # 10: direct channels + # 5: one and only private channel can be "Notifications". + 'read_only': boolean, + # true if this is a read-only subscription to a broadcast channel + # false if it's a public chat room + + 'actions':[('action name', 'view name'),] },] } """ @@ -233,7 +248,8 @@ def list_channels(current): {'name': sbs.channel.name or ('Notifications' if sbs.channel.is_private() else ''), 'key': sbs.channel.key, 'type': sbs.channel.typ, - 'actions': sbs.channel.get_actions_for(current.user), + 'read_only': sbs.read_only, + 'actions': sbs.get_actions(), 'unread': sbs.unread_count()} for sbs in current.user.subscriptions if sbs.is_visible] @@ -279,8 +295,8 @@ def add_members(current): { 'view':'_zops_add_members', 'channel_key': key, - 'read_only': True, # True if this is a Broadcast channel, - # False if it's a normal chat room + 'read_only': boolean, # true if this is a Broadcast channel, + # false if it's a normal chat room 'members': [key, key], } @@ -323,6 +339,9 @@ def add_unit_to_channel(current): 'view':'_zops_add_unit_to_channel', 'unit_key': key, 'channel_key': key, + 'read_only': boolean, # true if this is a Broadcast channel, + # false if it's a normal chat room + } # response: @@ -333,9 +352,11 @@ def add_unit_to_channel(current): 'code': 201 } """ + read_only = current.input['read_only'] newly_added, existing = [], [] for member_key in UnitModel.get_user_keys(current, current.input['unit_key']): sb, new = Subscriber(current).objects.get_or_create(user_id=member_key, + read_only=read_only, channel_id=current.input['channel_key']) if new: newly_added.append(member_key) @@ -437,7 +458,6 @@ def create_direct_channel(current): } - def find_message(current): """ Search in messages. If "channel_key" given, search will be limited to that channel, @@ -486,6 +506,88 @@ def find_message(current): current.output['results'].append(msg.serialize_for(current.user)) +def delete_channel(current): + """ + Delete a channel + + .. code-block:: python + + # request: + { + 'view':'_zops_delete_channel, + 'channel_key': key, + } + + # response: + { + 'status': 'OK', + 'code': 200 + } + """ + try: + Channel(current).objects.get(owner_id=current.user_id, + key=current.input['channel_key']).delete() + current.output = {'status': 'Deleted', 'code': 200} + except ObjectDoesNotExist: + raise HTTPError(404, "") + + +def edit_channel(current): + """ + Update channel name or description + + .. code-block:: python + + # request: + { + 'view':'_zops_edit_channel, + 'channel_key': key, + 'name': string, + 'description': string, + } + + # response: + { + 'status': 'OK', + 'code': 200 + } + """ + try: + Channel(current).objects.filter(owner_id=current.user_id, + key=current.input['channel_key'] + ).update(name=current.input['name'], + description=current.input['description']) + current.output = {'status': 'OK', 'code': 200} + except ObjectDoesNotExist: + raise HTTPError(404, "") + + +def pin_channel(current): + """ + Pin a channel to top of channel list + + .. code-block:: python + + # request: + { + 'view':'_zops_pin_channel, + 'channel_key': key, + } + + # response: + { + 'status': 'OK', + 'code': 200 + } + """ + try: + Subscriber(current).objects.get(user_id=current.user_id, + channel_id=current.input['channel_key']).update(pinned=True) + current.output = {'status': 'OK', 'code': 200} + except ObjectDoesNotExist: + raise HTTPError(404, "") + + def delete_message(current): """ Delete a message diff --git a/zengine/models/auth.py b/zengine/models/auth.py index 650abaeb..2df93ed7 100644 --- a/zengine/models/auth.py +++ b/zengine/models/auth.py @@ -30,9 +30,13 @@ class Meta: def __unicode__(self): return '%s' % self.name + @classmethod - def get_user_keys(cls, current, unit_key): - return User(current).objects.filter(unit_id=unit_key).values_list('key', flatten=True) + def get_user_keys(cls, unit_key): + stack = User.objects.filter(unit_id=unit_key).values_list('key', flatten=True) + for unit_key in cls.objects.filter(parent_id=unit_key).values_list('key', flatten=True): + stack.extend(cls.get_user_keys(unit_key)) + return stack diff --git a/zengine/settings.py b/zengine/settings.py index e81ce809..29347bfa 100644 --- a/zengine/settings.py +++ b/zengine/settings.py @@ -138,6 +138,9 @@ '_zops_add_to_favorites': 'zengine.messaging.views.add_to_favorites', '_zops_remove_from_favorites': 'zengine.messaging.views.remove_from_favorites', '_zops_list_favorites': 'zengine.messaging.views.list_favorites', + '_zops_edit_channel': 'zengine.messaging.views.edit_channel', + '_zops_delete_channel': 'zengine.messaging.views.delete_channel', + '_zops_pin_channel': 'zengine.messaging.views.pin_channel', # '_zops_': 'zengine.messaging.views.', } From 91d330cb31eedef54a0beaa8f28d69548abfb948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Wed, 13 Jul 2016 15:18:28 +0300 Subject: [PATCH 29/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/model.py | 20 ++++++++++++++++---- zengine/messaging/views.py | 36 +++++++++++++++++++++++++++++++++++- zengine/readthedocs.yml | 5 ----- zengine/settings.py | 1 + 4 files changed, 52 insertions(+), 10 deletions(-) delete mode 100644 zengine/readthedocs.yml diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 093afef1..a2c22f9b 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -252,16 +252,16 @@ class Message(Model): def get_actions_for(self, user): actions = [ - ('Favorite', 'favorite_message') + ('Favorite', '_zops_favorite_message') ] if self.sender == user: actions.extend([ - ('Delete', 'zops_delete_message'), - ('Edit', 'zops_edit_message') + ('Delete', '_zops_delete_message'), + ('Edit', '_zops_edit_message') ]) elif user: actions.extend([ - ('Flag', 'flag_message') + ('Flag', '_zops_flag_message') ]) def serialize_for(self, user=None): @@ -328,3 +328,15 @@ def pre_creation(self): self.channel = self.message.channel self.summary = self.message.body[:60] self.channel_name = self.channel.name + + +class FlaggedMessage(Model): + """ + A model to store users bookmarked messages + """ + channel = Channel() + user = UserModel() + message = Message() + + def pre_creation(self): + self.channel = self.message.channel diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index c30aec52..fe0948bc 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -10,7 +10,8 @@ from pyoko.exceptions import ObjectDoesNotExist from pyoko.lib.utils import get_object_from_path from zengine.lib.exceptions import HTTPError -from zengine.messaging.model import Channel, Attachment, Subscriber, Message, Favorite +from zengine.messaging.model import Channel, Attachment, Subscriber, Message, Favorite, \ + FlaggedMessage UserModel = get_object_from_path(settings.USER_MODEL) UnitModel = get_object_from_path(settings.UNIT_MODEL) @@ -642,6 +643,39 @@ def edit_message(current): raise HTTPError(404, "") +def flag_message(current): + """ + Flag inappropriate messages + + .. code-block:: python + + # request: + { + 'view':'_zops_flag_message', + 'message': { + 'key': key + 'flag': boolean, # true for flagging + # false for unflagging + } + } + # response: + { + ' + 'status': string, # 'OK' for success + 'code': int, # 200 for success + } + + """ + current.output = {'status': 'OK', 'code': 200} + if current.input['flag']: + FlaggedMessage.objects.get_or_create(current, + user_id=current.user_id, + message_id=current.input['key']) + else: + FlaggedMessage(current).objects.filter(user_id=current.user_id, + message_id=current.input['key']).delete() + + def get_message_actions(current): """ Returns applicable actions for current user for given message key diff --git a/zengine/readthedocs.yml b/zengine/readthedocs.yml deleted file mode 100644 index 5568dc78..00000000 --- a/zengine/readthedocs.yml +++ /dev/null @@ -1,5 +0,0 @@ -requirements_file: requirements/default.txt - -python: - version: 2 - pip_install: true diff --git a/zengine/settings.py b/zengine/settings.py index 29347bfa..614184fe 100644 --- a/zengine/settings.py +++ b/zengine/settings.py @@ -141,6 +141,7 @@ '_zops_edit_channel': 'zengine.messaging.views.edit_channel', '_zops_delete_channel': 'zengine.messaging.views.delete_channel', '_zops_pin_channel': 'zengine.messaging.views.pin_channel', + '_zops_flag_message': 'zengine.messaging.views.flag_message', # '_zops_': 'zengine.messaging.views.', } From f71fce86c1d0888e25ab96737630a12478602824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Wed, 13 Jul 2016 16:42:38 +0300 Subject: [PATCH 30/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/management_commands.py | 3 +-- zengine/messaging/model.py | 43 +++++++++++++++++++++++++++------- zengine/messaging/views.py | 39 ++++++++++++++++++++---------- zengine/settings.py | 2 +- 4 files changed, 63 insertions(+), 24 deletions(-) diff --git a/zengine/management_commands.py b/zengine/management_commands.py index 86e2221c..a82e97a1 100644 --- a/zengine/management_commands.py +++ b/zengine/management_commands.py @@ -207,8 +207,7 @@ def create_user_channels(self): ch, new = Channel.objects.get_or_create(owner=usr, typ=5) print("%s exchange: %s" % ('created' if new else 'existing', ch.code_name)) # create notification subscription to private exchange - sb, new = Subscriber.objects.get_or_create(channel=ch, user=usr, is_visible=True, - can_leave=False, inform_me=False) + sb, new = Subscriber.objects.get_or_create(channel=ch, user=usr, read_only=True) print("%s notify sub: %s" % ('created' if new else 'existing', ch.code_name)) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index a2c22f9b..f91f5d88 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -88,13 +88,14 @@ def get_or_create_direct_channel(cls, initiator_key, receiver_key): return channel @classmethod - def add_message(cls, channel_key, body, title=None, sender=None, url=None, typ=2, receiver=None): + def add_message(cls, channel_key, body, title=None, sender=None, url=None, typ=2, + receiver=None): mq_channel = cls._connect_mq() msg_object = Message(sender=sender, body=body, msg_title=title, url=url, typ=typ, channel_id=channel_key, receiver=receiver, key=uuid4().hex) mq_channel.basic_publish(exchange=channel_key, routing_key='', - body=json.dumps(msg_object.serialize_for())) + body=json.dumps(msg_object.serialize())) return msg_object.save() def get_last_messages(self): @@ -117,8 +118,6 @@ def create_exchange(self): exchange_type='fanout', durable=True) - - def pre_creation(self): if not self.code_name: if self.name: @@ -167,12 +166,14 @@ def _connect_mq(cls): def get_actions(self): actions = [ - ('Pin', 'pin_channel') + ('Pin', '_zops_pin_channel') ] - if self.channel.owner == self.user: + if self.channel.owner == self.user or self.can_manage: actions.extend([ ('Delete', '_zops_delete_channel'), ('Edit', '_zops_edit_channel') + ('Add Users', '_zops_add_members') + ('Add Unit', '_zops_add_unit_to_channel') ]) def unread_count(self): @@ -264,11 +265,25 @@ def get_actions_for(self, user): ('Flag', '_zops_flag_message') ]) - def serialize_for(self, user=None): + def serialize(self, user=None): + """ + Serializes message for given user. + + Note: + Should be called before first save(). Otherwise "is_update" will get wrong value. + + Args: + user: User object + + Returns: + Dict. JSON serialization ready dictionary object + """ return { 'content': self.body, 'type': self.typ, - 'time': self.updated_at, + 'updated_at': self.updated_at, + 'timestamp': self.timestamp, + 'is_update': self.exist, 'attachments': [attachment.serialize() for attachment in self.attachment_set], 'title': self.msg_title, 'sender_name': self.sender.full_name, @@ -281,6 +296,18 @@ def __unicode__(self): content = self.msg_title or self.body return "%s%s" % (content[:30], '...' if len(content) > 30 else '') + def _republish(self): + """ + Re-publishes updated message + """ + mq_channel = self.channel._connect_mq() + mq_channel.basic_publish(exchange=self.channel.key, routing_key='', + body=json.dumps(self.serialize())) + + def pre_save(self): + if self.exist: + self._republish() + ATTACHMENT_TYPES = ( (1, "Document"), diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index fe0948bc..c9b4c964 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -22,7 +22,10 @@ MSG_DICT = {'content': string, 'title': string, - 'time': datetime, + 'timestamp': datetime, + 'updated_at': datetime, + 'is_update': boolean, # false for new messages + # true if this is an updated message 'channel_key': key, 'sender_name': string, 'sender_key': key, @@ -90,7 +93,6 @@ def create_message(current): 'view':'_zops_create_message', 'message': { 'channel': key, # of channel - 'receiver': key, # of receiver. Should be set only for direct messages, 'body': string, # message text., 'type': int, # zengine.messaging.model.MSG_TYPES, 'attachments': [{ @@ -100,8 +102,8 @@ def create_message(current): }]} # response: { - 'status': string, # 'OK' for success - 'code': int, # 201 for success + 'status': 'Created', + 'code': 201, 'msg_key': key, # key of the message object, } @@ -143,6 +145,8 @@ def show_channel(current): 'avatar_url': string, }], 'last_messages': [MSG_DICT] + 'status': 'OK', + 'code': 200 } """ ch = Channel(current).objects.get(current.input['channel_key']) @@ -153,7 +157,7 @@ def show_channel(current): 'is_online': sb.user.is_online(), 'avatar_url': sb.user.get_avatar_url() } for sb in ch.subscriber_set], - 'last_messages': [msg.serialize_for(current.user) + 'last_messages': [msg.serialize(current.user) for msg in ch.get_last_messages()] } @@ -182,15 +186,19 @@ def channel_history(current): 'status': 'OK', 'code': 201, 'messages': [ - msg.serialize_for(current.user) + msg.serialize(current.user) for msg in Message.objects.filter(channel_id=current.input['channel_key'], timestamp__lt=current.input['timestamp'])[:20]] } -def last_seen_msg(current): +def report_last_seen_message(current): """ - Push timestamp of last seen message for a channel + Push timestamp of latest message of an ACTIVE channel. + + This view should be called with timestamp of latest message; + - When user opens (clicks on) a channel. + - Periodically (eg: setInterval for 15secs) while user staying in a channel. .. code-block:: python @@ -236,7 +244,7 @@ def list_channels(current): # 15: public channels (chat room/broadcast channel distinction comes from "read_only" flag) # 10: direct channels - # 5: one and only private channel can be "Notifications". + # 5: one and only private channel which can be "Notifications" 'read_only': boolean, # true if this is a read-only subscription to a broadcast channel # false if it's a public chat room @@ -259,6 +267,8 @@ def create_channel(current): """ Create a public channel. Can be a broadcast channel or normal chat room. + Chat room and broadcast distinction will be made at user subscription phase. + .. code-block:: python # request: @@ -297,7 +307,7 @@ def add_members(current): 'view':'_zops_add_members', 'channel_key': key, 'read_only': boolean, # true if this is a Broadcast channel, - # false if it's a normal chat room + # false if it's a normal chat room 'members': [key, key], } @@ -504,7 +514,7 @@ def find_message(current): query_set, pagination_data = _paginate(current_page=current.input['page'], query_set=query_set) current.output['pagination'] = pagination_data for msg in query_set: - current.output['results'].append(msg.serialize_for(current.user)) + current.output['results'].append(msg.serialize(current.user)) def delete_channel(current): @@ -638,8 +648,11 @@ def edit_message(current): """ current.output = {'status': 'OK', 'code': 200} msg = current.input['message'] - if not Message(current).objects.filter(sender_id=current.user_id, - key=msg['key']).update(body=msg['body']): + try: + msg = Message(current).objects.get(sender_id=current.user_id, key=msg['key']) + msg.body = msg['body'] + msg.save() + except ObjectDoesNotExist: raise HTTPError(404, "") diff --git a/zengine/settings.py b/zengine/settings.py index 614184fe..d6ab49cc 100644 --- a/zengine/settings.py +++ b/zengine/settings.py @@ -124,7 +124,7 @@ '_zops_show_channel': 'zengine.messaging.views.show_channel', '_zops_list_channels': 'zengine.messaging.views.list_channels', '_zops_channel_history': 'zengine.messaging.views.channel_history', - '_zops_last_seen_msg': 'zengine.messaging.views.last_seen_msg', + '_zops_report_last_seen_message': 'zengine.messaging.views.report_last_seen_message', '_zops_create_channel': 'zengine.messaging.views.create_channel', '_zops_add_members': 'zengine.messaging.views.add_members', '_zops_add_unit_to_channel': 'zengine.messaging.views.add_unit_to_channel', From 73c0c323ba7ea5b196b976e03818fbd1569a91bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Fri, 15 Jul 2016 15:17:39 +0300 Subject: [PATCH 31/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/lib/concurrent_amqp_test_client.py | 493 +++++++++++++++++++++ zengine/messaging/model.py | 22 +- zengine/messaging/views.py | 4 +- zengine/tornado_server/ws_to_queue.py | 12 +- 4 files changed, 517 insertions(+), 14 deletions(-) create mode 100644 zengine/lib/concurrent_amqp_test_client.py diff --git a/zengine/lib/concurrent_amqp_test_client.py b/zengine/lib/concurrent_amqp_test_client.py new file mode 100644 index 00000000..161a1d50 --- /dev/null +++ b/zengine/lib/concurrent_amqp_test_client.py @@ -0,0 +1,493 @@ +# -*- coding: utf-8 -*- +""" +When a user is not online, AMQP messages that sent to this +user are discarded at users private exchange. +When user come back, offline sent messages will be loaded +from DB and send to users private exchange. + +Because of this, we need to fully simulate 2 online users to test real time chat behaviour. + +""" + +# Copyright (C) 2015 ZetaOps Inc. +# +# This file is licensed under the GNU General Public License v3 +# (GPLv3). See LICENSE.txt for details. +import uuid + +import pika +from tornado.escape import json_encode, json_decode + +from zengine.current import Current +from zengine.lib.cache import Session +from zengine.log import log +from zengine.tornado_server.ws_to_queue import QueueManager, NON_BLOCKING_MQ_PARAMS +import sys + +from zengine.views.auth import Login + +sys.sessid_to_userid = {} +from ulakbus.models import User + +class TestQueueManager(QueueManager): + + def on_input_queue_declare(self, queue): + """ + AMQP connection callback. + Creates input channel. + + Args: + connection: AMQP connection + """ + log.info("input queue declared") + super(TestQueueManager, self).on_input_queue_declare(queue) + self.run_after_connection() + + # def connect(self): + # """ + # Creates connection to RabbitMQ server + # """ + # if self.connecting: + # log.info('PikaClient: Already connecting to RabbitMQ') + # return + # + # log.info('PikaClient: Connecting to RabbitMQ') + # self.connecting = True + # self.connection = pika.LibevConnection(NON_BLOCKING_MQ_PARAMS, + # on_open_error_callback=self.conn_err, + # on_open_callback=self.on_connected) + # print(self.connection.is_open) + + def __init__(self, *args, **kwargs): + super(TestQueueManager, self).__init__(*args, **kwargs) + log.info("queue manager init") + print("foo") + self.test_class = lambda qm: 1 + + def conn_err(self, *args, **kwargs): + log("conne err: %s %s" % (args, kwargs)) + + + + def run_after_connection(self): + log.info("run after connect") + self.test_class(self) + + def set_test_class(self, kls): + log.info("test class setted %s" % kls) + self.test_class = kls + + +class TestWebSocket(object): + def __init__(self, queue_manager, username, sess_id=None): + self.message_callbacks = {} + self.message_stack = {} + self.user = User.objects.get(username=username) + + self.request = type('MockWSRequestObject', (object,), {'remote_ip': '127.0.0.1'}) + self.queue_manager = queue_manager + self.sess_id = sess_id or uuid.uuid4().hex + sys.sessid_to_userid[self.sess_id] = self.user.key.lower() + self.queue_manager.register_websocket(self.sess_id, self) + self.login_user() + # mimic tornado ws object + # zengine.tornado_server.ws_to_queue.QueueManager will call write_message() method + self.write_message = self.backend_to_client + + def login_user(self): + session = Session(self.sess_id) + current = Current(session=session, input={}) + current.auth.set_user(self.user) + Login(current)._do_upgrade() + + def _get_sess_id(self): + return self.sess_id + + def backend_to_client(self, body): + """ + from backend to client + """ + body = json_decode(body) + try: + self.message_callbacks[body['callbackID']](body) + except KeyError: + self.message_stack[body['callbackID']] = body + log.info("WRITE MESSAGE TO CLIENT:\n%s" % (body,)) + print(body) + + def client_to_backend(self, message, callback): + """ + from client to backend + """ + cbid = uuid.uuid4().hex + self.message_callbacks[cbid] = callback + message = json_encode({"callbackID": cbid, "data": message}) + log.info("GOT MESSAGE FOR BACKEND %s: %s" % (self._get_sess_id(), message)) + self.queue_manager.redirect_incoming_message(self._get_sess_id(), message, self.request) + + +class ConcurrentTestCase(object): + def __init__(self, queue_manager): + log.info("ConcurrentTestCase class init with %s" % queue_manager) + self.queue_manager = queue_manager + + self.ws1 = TestWebSocket(self.queue_manager, 'ulakbus') + self.ws2 = TestWebSocket(self.queue_manager, 'ogrenci_isleri_1') + self.test_messaging_channel_list() + + def success_test_callback(self, response, request=None): + print(response) + assert response['code'] in (200, 201), "Process response not successful: \n %s \n %s" % ( + response, request + ) + + def test_messaging_channel_list(self): + + + + self.ws1.client_to_backend({"view": "_zops_list_channels"}, self.success_test_callback) + + +class AMQPLoop(object): + """This is an example consumer that will handle unexpected interactions + with RabbitMQ such as channel and connection closures. + + If RabbitMQ closes the connection, it will reopen it. You should + look at the output, as there are limited reasons why the connection may + be closed, which usually are tied to permission related issues or + socket timeouts. + + If the channel is closed, it will indicate a problem with one of the + commands that were issued and that should surface in the output as well. + + """ + EXCHANGE = 'message' + EXCHANGE_TYPE = 'topic' + QUEUE = 'text' + ROUTING_KEY = 'example.text' + + def __init__(self, amqp_url): + """Create a new instance of the consumer class, passing in the AMQP + URL used to connect to RabbitMQ. + + :param str amqp_url: The AMQP url to connect with + + """ + self._connection = None + self._channel = None + self._closing = False + self._consumer_tag = None + self._url = amqp_url + + def connect(self): + """This method connects to RabbitMQ, returning the connection handle. + When the connection is established, the on_connection_open method + will be invoked by pika. + + :rtype: pika.SelectConnection + + """ + log.info('Connecting to %s', self._url) + return pika.SelectConnection(pika.URLParameters(self._url), + self.on_connection_open, + stop_ioloop_on_close=False) + + def on_connection_open(self, unused_connection): + """This method is called by pika once the connection to RabbitMQ has + been established. It passes the handle to the connection object in + case we need it, but in this case, we'll just mark it unused. + + :type unused_connection: pika.SelectConnection + + """ + log.info('Connection opened') + self.add_on_connection_close_callback() + self.open_channel() + + def add_on_connection_close_callback(self): + """This method adds an on close callback that will be invoked by pika + when RabbitMQ closes the connection to the publisher unexpectedly. + + """ + log.info('Adding connection close callback') + self._connection.add_on_close_callback(self.on_connection_closed) + + def on_connection_closed(self, connection, reply_code, reply_text): + """This method is invoked by pika when the connection to RabbitMQ is + closed unexpectedly. Since it is unexpected, we will reconnect to + RabbitMQ if it disconnects. + + :param pika.connection.Connection connection: The closed connection obj + :param int reply_code: The server provided reply_code if given + :param str reply_text: The server provided reply_text if given + + """ + self._channel = None + if self._closing: + self._connection.ioloop.stop() + else: + log.warning('Connection closed, reopening in 5 seconds: (%s) %s', + reply_code, reply_text) + self._connection.add_timeout(5, self.reconnect) + + def reconnect(self): + """Will be invoked by the IOLoop timer if the connection is + closed. See the on_connection_closed method. + + """ + # This is the old connection IOLoop instance, stop its ioloop + self._connection.ioloop.stop() + + if not self._closing: + # Create a new connection + self._connection = self.connect() + + # There is now a new connection, needs a new ioloop to run + self._connection.ioloop.start() + + def open_channel(self): + """Open a new channel with RabbitMQ by issuing the Channel.Open RPC + command. When RabbitMQ responds that the channel is open, the + on_channel_open callback will be invoked by pika. + + """ + log.info('Creating a new channel') + self._connection.channel(on_open_callback=self.on_channel_open) + + def on_channel_open(self, channel): + """This method is invoked by pika when the channel has been opened. + The channel object is passed in so we can make use of it. + + Since the channel is now open, we'll declare the exchange to use. + + :param pika.channel.Channel channel: The channel object + + """ + log.info('Channel opened') + self._channel = channel + self.add_on_channel_close_callback() + self.setup_exchange(self.EXCHANGE) + + def add_on_channel_close_callback(self): + """This method tells pika to call the on_channel_closed method if + RabbitMQ unexpectedly closes the channel. + + """ + log.info('Adding channel close callback') + self._channel.add_on_close_callback(self.on_channel_closed) + + def on_channel_closed(self, channel, reply_code, reply_text): + """Invoked by pika when RabbitMQ unexpectedly closes the channel. + Channels are usually closed if you attempt to do something that + violates the protocol, such as re-declare an exchange or queue with + different parameters. In this case, we'll close the connection + to shutdown the object. + + :param pika.channel.Channel: The closed channel + :param int reply_code: The numeric reason the channel was closed + :param str reply_text: The text reason the channel was closed + + """ + log.warning('Channel %i was closed: (%s) %s', + channel, reply_code, reply_text) + self._connection.close() + + def setup_exchange(self, exchange_name): + """Setup the exchange on RabbitMQ by invoking the Exchange.Declare RPC + command. When it is complete, the on_exchange_declareok method will + be invoked by pika. + + :param str|unicode exchange_name: The name of the exchange to declare + + """ + log.info('Declaring exchange %s', exchange_name) + self._channel.exchange_declare(self.on_exchange_declareok, + exchange_name, + self.EXCHANGE_TYPE) + + def on_exchange_declareok(self, unused_frame): + """Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC + command. + + :param pika.Frame.Method unused_frame: Exchange.DeclareOk response frame + + """ + log.info('Exchange declared') + self.setup_queue(self.QUEUE) + + def setup_queue(self, queue_name): + """Setup the queue on RabbitMQ by invoking the Queue.Declare RPC + command. When it is complete, the on_queue_declareok method will + be invoked by pika. + + :param str|unicode queue_name: The name of the queue to declare. + + """ + log.info('Declaring queue %s', queue_name) + self._channel.queue_declare(self.on_queue_declareok, queue_name) + + def on_queue_declareok(self, method_frame): + """Method invoked by pika when the Queue.Declare RPC call made in + setup_queue has completed. In this method we will bind the queue + and exchange together with the routing key by issuing the Queue.Bind + RPC command. When this command is complete, the on_bindok method will + be invoked by pika. + + :param pika.frame.Method method_frame: The Queue.DeclareOk frame + + """ + log.info('Binding %s to %s with %s', + self.EXCHANGE, self.QUEUE, self.ROUTING_KEY) + self._channel.queue_bind(self.on_bindok, self.QUEUE, + self.EXCHANGE, self.ROUTING_KEY) + + def on_bindok(self, unused_frame): + """Invoked by pika when the Queue.Bind method has completed. At this + point we will start consuming messages by calling start_consuming + which will invoke the needed RPC commands to start the process. + + :param pika.frame.Method unused_frame: The Queue.BindOk response frame + + """ + log.info('Queue bound') + self.start_consuming() + + def start_consuming(self): + """This method sets up the consumer by first calling + add_on_cancel_callback so that the object is notified if RabbitMQ + cancels the consumer. It then issues the Basic.Consume RPC command + which returns the consumer tag that is used to uniquely identify the + consumer with RabbitMQ. We keep the value to use it when we want to + cancel consuming. The on_message method is passed in as a callback pika + will invoke when a message is fully received. + + """ + log.info('Issuing consumer related RPC commands') + self.add_on_cancel_callback() + self._consumer_tag = self._channel.basic_consume(self.on_message, + self.QUEUE) + + def add_on_cancel_callback(self): + """Add a callback that will be invoked if RabbitMQ cancels the consumer + for some reason. If RabbitMQ does cancel the consumer, + on_consumer_cancelled will be invoked by pika. + + """ + log.info('Adding consumer cancellation callback') + self._channel.add_on_cancel_callback(self.on_consumer_cancelled) + + def on_consumer_cancelled(self, method_frame): + """Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer + receiving messages. + + :param pika.frame.Method method_frame: The Basic.Cancel frame + + """ + log.info('Consumer was cancelled remotely, shutting down: %r', + method_frame) + if self._channel: + self._channel.close() + + def on_message(self, unused_channel, basic_deliver, properties, body): + """Invoked by pika when a message is delivered from RabbitMQ. The + channel is passed for your convenience. The basic_deliver object that + is passed in carries the exchange, routing key, delivery tag and + a redelivered flag for the message. The properties passed in is an + instance of BasicProperties with the message properties and the body + is the message that was sent. + + :param pika.channel.Channel unused_channel: The channel object + :param pika.Spec.Basic.Deliver: basic_deliver method + :param pika.Spec.BasicProperties: properties + :param str|unicode body: The message body + + """ + log.info('Received message # %s from %s: %s', + basic_deliver.delivery_tag, properties.app_id, body) + self.acknowledge_message(basic_deliver.delivery_tag) + + def acknowledge_message(self, delivery_tag): + """Acknowledge the message delivery from RabbitMQ by sending a + Basic.Ack RPC method for the delivery tag. + + :param int delivery_tag: The delivery tag from the Basic.Deliver frame + + """ + log.info('Acknowledging message %s', delivery_tag) + self._channel.basic_ack(delivery_tag) + + def stop_consuming(self): + """Tell RabbitMQ that you would like to stop consuming by sending the + Basic.Cancel RPC command. + + """ + if self._channel: + log.info('Sending a Basic.Cancel RPC command to RabbitMQ') + self._channel.basic_cancel(self.on_cancelok, self._consumer_tag) + + def on_cancelok(self, unused_frame): + """This method is invoked by pika when RabbitMQ acknowledges the + cancellation of a consumer. At this point we will close the channel. + This will invoke the on_channel_closed method once the channel has been + closed, which will in-turn close the connection. + + :param pika.frame.Method unused_frame: The Basic.CancelOk frame + + """ + log.info('RabbitMQ acknowledged the cancellation of the consumer') + self.close_channel() + + def close_channel(self): + """Call to close the channel with RabbitMQ cleanly by issuing the + Channel.Close RPC command. + + """ + log.info('Closing the channel') + self._channel.close() + + def run(self): + """Run the example consumer by connecting to RabbitMQ and then + starting the IOLoop to block and allow the SelectConnection to operate. + + """ + self._connection = self.connect() + self._connection.ioloop.start() + + def stop(self): + """Cleanly shutdown the connection to RabbitMQ by stopping the consumer + with RabbitMQ. When RabbitMQ confirms the cancellation, on_cancelok + will be invoked by pika, which will then closing the channel and + connection. The IOLoop is started again because this method is invoked + when CTRL-C is pressed raising a KeyboardInterrupt exception. This + exception stops the IOLoop which needs to be running for pika to + communicate with RabbitMQ. All of the commands issued prior to starting + the IOLoop will be buffered but not processed. + + """ + log.info('Stopping') + self._closing = True + self.stop_consuming() + self._connection.ioloop.start() + log.info('Stopped') + + def close_connection(self): + """This method closes the connection to RabbitMQ.""" + log.info('Closing connection') + self._connection.close() + + +def main(): + from tornado import ioloop + # initiate amqp manager + ioloop = ioloop.IOLoop.instance() + qm = TestQueueManager(io_loop=ioloop) + + # initiate test case + qm.set_test_class(ConcurrentTestCase) + + qm.connect() + ioloop.start() + + +if __name__ == '__main__': + main() diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index f91f5d88..46f9aa83 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -24,6 +24,8 @@ def get_mq_connection(): connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) channel = connection.channel() + if not channel.is_open: + channel.open() return connection, channel @@ -56,6 +58,9 @@ class Channel(Model): description = field.String("Description") owner = UserModel(reverse_name='created_channels', null=True) + def __unicode__(self): + return "%s (%s's %s channel)" % (self.name or '', self.owner.__unicode__(), self.get_typ_display()) + # # class Managers(ListNode): # user = UserModel() @@ -132,7 +137,7 @@ def pre_creation(self): else: self.key = self.code_name - def post_creation(self): + def post_save(self): self.create_exchange() @@ -158,6 +163,9 @@ class Subscriber(Model): # status = field.Integer("Status", choices=SUBSCRIPTION_STATUS) + def __unicode__(self): + return "%s >> %s" % (self.user.full_name, self.channel.__unicode__()) + @classmethod def _connect_mq(cls): if cls.mq_connection is None or cls.mq_connection.is_closed: @@ -171,15 +179,17 @@ def get_actions(self): if self.channel.owner == self.user or self.can_manage: actions.extend([ ('Delete', '_zops_delete_channel'), - ('Edit', '_zops_edit_channel') - ('Add Users', '_zops_add_members') + ('Edit', '_zops_edit_channel'), + ('Add Users', '_zops_add_members'), ('Add Unit', '_zops_add_unit_to_channel') ]) def unread_count(self): # FIXME: track and return actual unread message count - return self.channel.message_set.objects.filter( - timestamp__lt=self.last_seen_msg_time).count() + if self.last_seen_msg_time: + return self.channel.message_set.objects.filter(timestamp__lt=self.last_seen_msg_time).count() + else: + self.channel.message_set.objects.filter().count() def create_exchange(self): """ @@ -211,8 +221,6 @@ def post_creation(self): self.create_exchange() self.bind_to_channel() - def __unicode__(self): - return "%s in %s" % (self.user, self.channel.name) MSG_TYPES = ( diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index c9b4c964..d27024c7 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -244,7 +244,7 @@ def list_channels(current): # 15: public channels (chat room/broadcast channel distinction comes from "read_only" flag) # 10: direct channels - # 5: one and only private channel which can be "Notifications" + # 5: one and only private channel which is "Notifications" 'read_only': boolean, # true if this is a read-only subscription to a broadcast channel # false if it's a public chat room @@ -260,7 +260,7 @@ def list_channels(current): 'read_only': sbs.read_only, 'actions': sbs.get_actions(), 'unread': sbs.unread_count()} for sbs in - current.user.subscriptions if sbs.is_visible] + current.user.subscriptions.objects.filter(is_visible=True)] def create_channel(current): diff --git a/zengine/tornado_server/ws_to_queue.py b/zengine/tornado_server/ws_to_queue.py index 146a3d5f..cf8e9f4b 100644 --- a/zengine/tornado_server/ws_to_queue.py +++ b/zengine/tornado_server/ws_to_queue.py @@ -124,7 +124,7 @@ class QueueManager(object): """ INPUT_QUEUE_NAME = 'in_queue' - def __init__(self, io_loop): + def __init__(self, io_loop=None): log.info('PikaClient: __init__') self.io_loop = io_loop self.connected = False @@ -134,7 +134,7 @@ def __init__(self, io_loop): self.out_channels = {} self.out_channel = None self.websockets = {} - self.connect() + # self.connect() def connect(self): """ @@ -148,6 +148,8 @@ def connect(self): self.connecting = True self.connection = TornadoConnection(NON_BLOCKING_MQ_PARAMS, + stop_ioloop_on_close=False, + custom_ioloop=self.io_loop, on_open_callback=self.on_connected) def on_connected(self, connection): @@ -160,10 +162,9 @@ def on_connected(self, connection): """ log.info('PikaClient: connected to RabbitMQ') self.connected = True - self.connection = connection - self.in_channel = self.connection.channel(self.on_conn_open) + self.in_channel = self.connection.channel(self.on_channel_open) - def on_conn_open(self, channel): + def on_channel_open(self, channel): """ Input channel creation callback Queue declaration done here @@ -265,6 +266,7 @@ def on_message(self, channel, method, header, body): user_id = method.exchange[4:] log.debug("WS RPLY for %s: %s" % (user_id, body)) if user_id in self.websockets: + log.info("write msg to client") self.websockets[user_id].write_message(body) channel.basic_ack(delivery_tag=method.delivery_tag) elif 'sessid_to_userid' in body: From f8fa706750f98ea8ffc330f2fb789964594aac3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Mon, 18 Jul 2016 15:09:14 +0300 Subject: [PATCH 32/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/lib/concurrent_amqp_test_client.py | 26 +++++++++++++++------- zengine/messaging/views.py | 2 +- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/zengine/lib/concurrent_amqp_test_client.py b/zengine/lib/concurrent_amqp_test_client.py index 161a1d50..4b165d17 100644 --- a/zengine/lib/concurrent_amqp_test_client.py +++ b/zengine/lib/concurrent_amqp_test_client.py @@ -61,7 +61,6 @@ def on_input_queue_declare(self, queue): def __init__(self, *args, **kwargs): super(TestQueueManager, self).__init__(*args, **kwargs) log.info("queue manager init") - print("foo") self.test_class = lambda qm: 1 def conn_err(self, *args, **kwargs): @@ -113,7 +112,6 @@ def backend_to_client(self, body): except KeyError: self.message_stack[body['callbackID']] = body log.info("WRITE MESSAGE TO CLIENT:\n%s" % (body,)) - print(body) def client_to_backend(self, message, callback): """ @@ -133,19 +131,31 @@ def __init__(self, queue_manager): self.ws1 = TestWebSocket(self.queue_manager, 'ulakbus') self.ws2 = TestWebSocket(self.queue_manager, 'ogrenci_isleri_1') - self.test_messaging_channel_list() + self.run_tests() + + def run_tests(self): + for name in sorted(self.__class__.__dict__): + if name.startswith("test_"): + try: + getattr(self, name)() + print("%s succesfully passed" % name) + except: + print("%s FAIL" % name) def success_test_callback(self, response, request=None): - print(response) + # print(response) assert response['code'] in (200, 201), "Process response not successful: \n %s \n %s" % ( response, request ) - def test_messaging_channel_list(self): - - + def test_channel_list(self): + self.ws1.client_to_backend({"view": "_zops_list_channels"}, + self.success_test_callback) - self.ws1.client_to_backend({"view": "_zops_list_channels"}, self.success_test_callback) + def test_search_user(self): + self.ws1.client_to_backend({"view": "_zops_search_user", + "query":"x"}, + self.success_test_callback) class AMQPLoop(object): diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index d27024c7..1327e3ad 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -407,7 +407,7 @@ def search_user(current): 'status': 'OK', 'code': 201 } - for user in UserModel(current).objects.search_on(settings.MESSAGING_USER_SEARCH_FIELDS, + for user in UserModel(current).objects.search_on(*settings.MESSAGING_USER_SEARCH_FIELDS, contains=current.input['query']): current.output['results'].append((user.full_name, user.key, user.avatar)) From c65c976fbe720374834912dc57641a132b44cd21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Mon, 18 Jul 2016 15:41:39 +0300 Subject: [PATCH 33/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/lib/concurrent_amqp_test_client.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/zengine/lib/concurrent_amqp_test_client.py b/zengine/lib/concurrent_amqp_test_client.py index 4b165d17..e313267a 100644 --- a/zengine/lib/concurrent_amqp_test_client.py +++ b/zengine/lib/concurrent_amqp_test_client.py @@ -99,9 +99,6 @@ def login_user(self): current.auth.set_user(self.user) Login(current)._do_upgrade() - def _get_sess_id(self): - return self.sess_id - def backend_to_client(self, body): """ from backend to client @@ -120,8 +117,8 @@ def client_to_backend(self, message, callback): cbid = uuid.uuid4().hex self.message_callbacks[cbid] = callback message = json_encode({"callbackID": cbid, "data": message}) - log.info("GOT MESSAGE FOR BACKEND %s: %s" % (self._get_sess_id(), message)) - self.queue_manager.redirect_incoming_message(self._get_sess_id(), message, self.request) + log.info("GOT MESSAGE FOR BACKEND %s: %s" % (self.sess_id, message)) + self.queue_manager.redirect_incoming_message(self.sess_id, message, self.request) class ConcurrentTestCase(object): From 334a4dc5ed6eb2606e27bf390967d121cd8cb6c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Tue, 19 Jul 2016 13:53:17 +0300 Subject: [PATCH 34/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- .../activities/async_amqp/messaging_tests.py | 29 ++ zengine/lib/concurrent_amqp_test_client.py | 404 ++---------------- zengine/management_commands.py | 5 +- zengine/messaging/model.py | 19 +- zengine/messaging/views.py | 6 +- 5 files changed, 84 insertions(+), 379 deletions(-) create mode 100644 tests/activities/async_amqp/messaging_tests.py diff --git a/tests/activities/async_amqp/messaging_tests.py b/tests/activities/async_amqp/messaging_tests.py new file mode 100644 index 00000000..467f66e4 --- /dev/null +++ b/tests/activities/async_amqp/messaging_tests.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +""" +""" + +# Copyright (C) 2015 ZetaOps Inc. +# +# This file is licensed under the GNU General Public License v3 +# (GPLv3). See LICENSE.txt for details. +from zengine.lib.concurrent_amqp_test_client import ConcurrentTestCase + + +class TestCase(ConcurrentTestCase): + def test_channel_list(self): + self.ws1.client_to_backend({"view": "_zops_list_channels"}, + self.success_test_callback) + + def test_search_user(self): + self.ws1.client_to_backend({"view": "_zops_search_user", + "query":"x"}, + self.success_test_callback) + + + + + + + +if __name__ == '__main__': + TestCase() diff --git a/zengine/lib/concurrent_amqp_test_client.py b/zengine/lib/concurrent_amqp_test_client.py index e313267a..0406f1bf 100644 --- a/zengine/lib/concurrent_amqp_test_client.py +++ b/zengine/lib/concurrent_amqp_test_client.py @@ -18,6 +18,8 @@ import pika from tornado.escape import json_encode, json_decode +from pyoko.conf import settings +from pyoko.lib.utils import get_object_from_path from zengine.current import Current from zengine.lib.cache import Session from zengine.log import log @@ -27,10 +29,10 @@ from zengine.views.auth import Login sys.sessid_to_userid = {} -from ulakbus.models import User +UserModel = get_object_from_path(settings.USER_MODEL) -class TestQueueManager(QueueManager): +class TestQueueManager(QueueManager): def on_input_queue_declare(self, queue): """ AMQP connection callback. @@ -43,21 +45,6 @@ def on_input_queue_declare(self, queue): super(TestQueueManager, self).on_input_queue_declare(queue) self.run_after_connection() - # def connect(self): - # """ - # Creates connection to RabbitMQ server - # """ - # if self.connecting: - # log.info('PikaClient: Already connecting to RabbitMQ') - # return - # - # log.info('PikaClient: Connecting to RabbitMQ') - # self.connecting = True - # self.connection = pika.LibevConnection(NON_BLOCKING_MQ_PARAMS, - # on_open_error_callback=self.conn_err, - # on_open_callback=self.on_connected) - # print(self.connection.is_open) - def __init__(self, *args, **kwargs): super(TestQueueManager, self).__init__(*args, **kwargs) log.info("queue manager init") @@ -66,8 +53,6 @@ def __init__(self, *args, **kwargs): def conn_err(self, *args, **kwargs): log("conne err: %s %s" % (args, kwargs)) - - def run_after_connection(self): log.info("run after connect") self.test_class(self) @@ -77,14 +62,15 @@ def set_test_class(self, kls): self.test_class = kls -class TestWebSocket(object): +class TestWSClient(object): def __init__(self, queue_manager, username, sess_id=None): self.message_callbacks = {} self.message_stack = {} - self.user = User.objects.get(username=username) + self.user = UserModel.objects.get(username=username) self.request = type('MockWSRequestObject', (object,), {'remote_ip': '127.0.0.1'}) self.queue_manager = queue_manager + # order is important! self.sess_id = sess_id or uuid.uuid4().hex sys.sessid_to_userid[self.sess_id] = self.user.key.lower() self.queue_manager.register_websocket(self.sess_id, self) @@ -122,13 +108,34 @@ def client_to_backend(self, message, callback): class ConcurrentTestCase(object): + """ + Extend this class, define your test methods with "test_" prefix. + + + + """ + def __init__(self, queue_manager): + from tornado import ioloop log.info("ConcurrentTestCase class init with %s" % queue_manager) - self.queue_manager = queue_manager + ioloop = ioloop.IOLoop.instance() + self.ws1 = self.get_client('ulakbus') + self.ws2 = self.get_client('ogrenci_isleri_1') + self.queue_manager = TestQueueManager(io_loop=ioloop) + # initiate amqp manager + self.queue_manager.set_test_class(self.run_tests) + self.queue_manager.connect() + ioloop.start() + + def get_client(self, username): + """ + Args: + username: username for this client instance - self.ws1 = TestWebSocket(self.queue_manager, 'ulakbus') - self.ws2 = TestWebSocket(self.queue_manager, 'ogrenci_isleri_1') - self.run_tests() + Returns: + Logged in TestWSClient instance for given username + """ + return TestWSClient(self.queue_manager, username) def run_tests(self): for name in sorted(self.__class__.__dict__): @@ -151,350 +158,5 @@ def test_channel_list(self): def test_search_user(self): self.ws1.client_to_backend({"view": "_zops_search_user", - "query":"x"}, + "query": "x"}, self.success_test_callback) - - -class AMQPLoop(object): - """This is an example consumer that will handle unexpected interactions - with RabbitMQ such as channel and connection closures. - - If RabbitMQ closes the connection, it will reopen it. You should - look at the output, as there are limited reasons why the connection may - be closed, which usually are tied to permission related issues or - socket timeouts. - - If the channel is closed, it will indicate a problem with one of the - commands that were issued and that should surface in the output as well. - - """ - EXCHANGE = 'message' - EXCHANGE_TYPE = 'topic' - QUEUE = 'text' - ROUTING_KEY = 'example.text' - - def __init__(self, amqp_url): - """Create a new instance of the consumer class, passing in the AMQP - URL used to connect to RabbitMQ. - - :param str amqp_url: The AMQP url to connect with - - """ - self._connection = None - self._channel = None - self._closing = False - self._consumer_tag = None - self._url = amqp_url - - def connect(self): - """This method connects to RabbitMQ, returning the connection handle. - When the connection is established, the on_connection_open method - will be invoked by pika. - - :rtype: pika.SelectConnection - - """ - log.info('Connecting to %s', self._url) - return pika.SelectConnection(pika.URLParameters(self._url), - self.on_connection_open, - stop_ioloop_on_close=False) - - def on_connection_open(self, unused_connection): - """This method is called by pika once the connection to RabbitMQ has - been established. It passes the handle to the connection object in - case we need it, but in this case, we'll just mark it unused. - - :type unused_connection: pika.SelectConnection - - """ - log.info('Connection opened') - self.add_on_connection_close_callback() - self.open_channel() - - def add_on_connection_close_callback(self): - """This method adds an on close callback that will be invoked by pika - when RabbitMQ closes the connection to the publisher unexpectedly. - - """ - log.info('Adding connection close callback') - self._connection.add_on_close_callback(self.on_connection_closed) - - def on_connection_closed(self, connection, reply_code, reply_text): - """This method is invoked by pika when the connection to RabbitMQ is - closed unexpectedly. Since it is unexpected, we will reconnect to - RabbitMQ if it disconnects. - - :param pika.connection.Connection connection: The closed connection obj - :param int reply_code: The server provided reply_code if given - :param str reply_text: The server provided reply_text if given - - """ - self._channel = None - if self._closing: - self._connection.ioloop.stop() - else: - log.warning('Connection closed, reopening in 5 seconds: (%s) %s', - reply_code, reply_text) - self._connection.add_timeout(5, self.reconnect) - - def reconnect(self): - """Will be invoked by the IOLoop timer if the connection is - closed. See the on_connection_closed method. - - """ - # This is the old connection IOLoop instance, stop its ioloop - self._connection.ioloop.stop() - - if not self._closing: - # Create a new connection - self._connection = self.connect() - - # There is now a new connection, needs a new ioloop to run - self._connection.ioloop.start() - - def open_channel(self): - """Open a new channel with RabbitMQ by issuing the Channel.Open RPC - command. When RabbitMQ responds that the channel is open, the - on_channel_open callback will be invoked by pika. - - """ - log.info('Creating a new channel') - self._connection.channel(on_open_callback=self.on_channel_open) - - def on_channel_open(self, channel): - """This method is invoked by pika when the channel has been opened. - The channel object is passed in so we can make use of it. - - Since the channel is now open, we'll declare the exchange to use. - - :param pika.channel.Channel channel: The channel object - - """ - log.info('Channel opened') - self._channel = channel - self.add_on_channel_close_callback() - self.setup_exchange(self.EXCHANGE) - - def add_on_channel_close_callback(self): - """This method tells pika to call the on_channel_closed method if - RabbitMQ unexpectedly closes the channel. - - """ - log.info('Adding channel close callback') - self._channel.add_on_close_callback(self.on_channel_closed) - - def on_channel_closed(self, channel, reply_code, reply_text): - """Invoked by pika when RabbitMQ unexpectedly closes the channel. - Channels are usually closed if you attempt to do something that - violates the protocol, such as re-declare an exchange or queue with - different parameters. In this case, we'll close the connection - to shutdown the object. - - :param pika.channel.Channel: The closed channel - :param int reply_code: The numeric reason the channel was closed - :param str reply_text: The text reason the channel was closed - - """ - log.warning('Channel %i was closed: (%s) %s', - channel, reply_code, reply_text) - self._connection.close() - - def setup_exchange(self, exchange_name): - """Setup the exchange on RabbitMQ by invoking the Exchange.Declare RPC - command. When it is complete, the on_exchange_declareok method will - be invoked by pika. - - :param str|unicode exchange_name: The name of the exchange to declare - - """ - log.info('Declaring exchange %s', exchange_name) - self._channel.exchange_declare(self.on_exchange_declareok, - exchange_name, - self.EXCHANGE_TYPE) - - def on_exchange_declareok(self, unused_frame): - """Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC - command. - - :param pika.Frame.Method unused_frame: Exchange.DeclareOk response frame - - """ - log.info('Exchange declared') - self.setup_queue(self.QUEUE) - - def setup_queue(self, queue_name): - """Setup the queue on RabbitMQ by invoking the Queue.Declare RPC - command. When it is complete, the on_queue_declareok method will - be invoked by pika. - - :param str|unicode queue_name: The name of the queue to declare. - - """ - log.info('Declaring queue %s', queue_name) - self._channel.queue_declare(self.on_queue_declareok, queue_name) - - def on_queue_declareok(self, method_frame): - """Method invoked by pika when the Queue.Declare RPC call made in - setup_queue has completed. In this method we will bind the queue - and exchange together with the routing key by issuing the Queue.Bind - RPC command. When this command is complete, the on_bindok method will - be invoked by pika. - - :param pika.frame.Method method_frame: The Queue.DeclareOk frame - - """ - log.info('Binding %s to %s with %s', - self.EXCHANGE, self.QUEUE, self.ROUTING_KEY) - self._channel.queue_bind(self.on_bindok, self.QUEUE, - self.EXCHANGE, self.ROUTING_KEY) - - def on_bindok(self, unused_frame): - """Invoked by pika when the Queue.Bind method has completed. At this - point we will start consuming messages by calling start_consuming - which will invoke the needed RPC commands to start the process. - - :param pika.frame.Method unused_frame: The Queue.BindOk response frame - - """ - log.info('Queue bound') - self.start_consuming() - - def start_consuming(self): - """This method sets up the consumer by first calling - add_on_cancel_callback so that the object is notified if RabbitMQ - cancels the consumer. It then issues the Basic.Consume RPC command - which returns the consumer tag that is used to uniquely identify the - consumer with RabbitMQ. We keep the value to use it when we want to - cancel consuming. The on_message method is passed in as a callback pika - will invoke when a message is fully received. - - """ - log.info('Issuing consumer related RPC commands') - self.add_on_cancel_callback() - self._consumer_tag = self._channel.basic_consume(self.on_message, - self.QUEUE) - - def add_on_cancel_callback(self): - """Add a callback that will be invoked if RabbitMQ cancels the consumer - for some reason. If RabbitMQ does cancel the consumer, - on_consumer_cancelled will be invoked by pika. - - """ - log.info('Adding consumer cancellation callback') - self._channel.add_on_cancel_callback(self.on_consumer_cancelled) - - def on_consumer_cancelled(self, method_frame): - """Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer - receiving messages. - - :param pika.frame.Method method_frame: The Basic.Cancel frame - - """ - log.info('Consumer was cancelled remotely, shutting down: %r', - method_frame) - if self._channel: - self._channel.close() - - def on_message(self, unused_channel, basic_deliver, properties, body): - """Invoked by pika when a message is delivered from RabbitMQ. The - channel is passed for your convenience. The basic_deliver object that - is passed in carries the exchange, routing key, delivery tag and - a redelivered flag for the message. The properties passed in is an - instance of BasicProperties with the message properties and the body - is the message that was sent. - - :param pika.channel.Channel unused_channel: The channel object - :param pika.Spec.Basic.Deliver: basic_deliver method - :param pika.Spec.BasicProperties: properties - :param str|unicode body: The message body - - """ - log.info('Received message # %s from %s: %s', - basic_deliver.delivery_tag, properties.app_id, body) - self.acknowledge_message(basic_deliver.delivery_tag) - - def acknowledge_message(self, delivery_tag): - """Acknowledge the message delivery from RabbitMQ by sending a - Basic.Ack RPC method for the delivery tag. - - :param int delivery_tag: The delivery tag from the Basic.Deliver frame - - """ - log.info('Acknowledging message %s', delivery_tag) - self._channel.basic_ack(delivery_tag) - - def stop_consuming(self): - """Tell RabbitMQ that you would like to stop consuming by sending the - Basic.Cancel RPC command. - - """ - if self._channel: - log.info('Sending a Basic.Cancel RPC command to RabbitMQ') - self._channel.basic_cancel(self.on_cancelok, self._consumer_tag) - - def on_cancelok(self, unused_frame): - """This method is invoked by pika when RabbitMQ acknowledges the - cancellation of a consumer. At this point we will close the channel. - This will invoke the on_channel_closed method once the channel has been - closed, which will in-turn close the connection. - - :param pika.frame.Method unused_frame: The Basic.CancelOk frame - - """ - log.info('RabbitMQ acknowledged the cancellation of the consumer') - self.close_channel() - - def close_channel(self): - """Call to close the channel with RabbitMQ cleanly by issuing the - Channel.Close RPC command. - - """ - log.info('Closing the channel') - self._channel.close() - - def run(self): - """Run the example consumer by connecting to RabbitMQ and then - starting the IOLoop to block and allow the SelectConnection to operate. - - """ - self._connection = self.connect() - self._connection.ioloop.start() - - def stop(self): - """Cleanly shutdown the connection to RabbitMQ by stopping the consumer - with RabbitMQ. When RabbitMQ confirms the cancellation, on_cancelok - will be invoked by pika, which will then closing the channel and - connection. The IOLoop is started again because this method is invoked - when CTRL-C is pressed raising a KeyboardInterrupt exception. This - exception stops the IOLoop which needs to be running for pika to - communicate with RabbitMQ. All of the commands issued prior to starting - the IOLoop will be buffered but not processed. - - """ - log.info('Stopping') - self._closing = True - self.stop_consuming() - self._connection.ioloop.start() - log.info('Stopped') - - def close_connection(self): - """This method closes the connection to RabbitMQ.""" - log.info('Closing connection') - self._connection.close() - - -def main(): - from tornado import ioloop - # initiate amqp manager - ioloop = ioloop.IOLoop.instance() - qm = TestQueueManager(io_loop=ioloop) - - # initiate test case - qm.set_test_class(ConcurrentTestCase) - - qm.connect() - ioloop.start() - - -if __name__ == '__main__': - main() diff --git a/zengine/management_commands.py b/zengine/management_commands.py index a82e97a1..9229987f 100644 --- a/zengine/management_commands.py +++ b/zengine/management_commands.py @@ -207,7 +207,10 @@ def create_user_channels(self): ch, new = Channel.objects.get_or_create(owner=usr, typ=5) print("%s exchange: %s" % ('created' if new else 'existing', ch.code_name)) # create notification subscription to private exchange - sb, new = Subscriber.objects.get_or_create(channel=ch, user=usr, read_only=True) + sb, new = Subscriber.objects.get_or_create(channel=ch, + user=usr, + read_only=True, + name='Notifications') print("%s notify sub: %s" % ('created' if new else 'existing', ch.code_name)) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 46f9aa83..17002b1f 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -78,19 +78,24 @@ def get_or_create_direct_channel(cls, initiator_key, receiver_key): receiver: User, other party Returns: - Channel + (Channel, receiver_name) """ existing = cls.objects.OR().filter( code_name='%s_%s' % (initiator_key, receiver_key)).filter( code_name='%s_%s' % (receiver_key, initiator_key)) + receiver_name = UserModel.objects.get(receiver_key).full_name if existing: - return existing[0] + return existing[0], receiver_name else: channel_name = '%s_%s' % (initiator_key, receiver_key) channel = cls(is_direct=True, code_name=channel_name).save() - Subscriber(channel=channel, user_id=initiator_key).save() - Subscriber(channel=channel, user_id=receiver_key).save() - return channel + Subscriber(channel=channel, + user_id=initiator_key, + name=receiver_name).save() + Subscriber(channel=channel, + user_id=receiver_key, + name=UserModel.objects.get(initiator_key).full_name).save() + return channel, receiver_name @classmethod def add_message(cls, channel_key, body, title=None, sender=None, url=None, typ=2, @@ -150,6 +155,7 @@ class Subscriber(Model): mq_connection = None channel = Channel() + name = field.String("Subscription Name") user = UserModel(reverse_name='subscriptions') is_muted = field.Boolean("Mute the channel", default=False) pinned = field.Boolean("Pin channel to top", default=False) @@ -221,6 +227,9 @@ def post_creation(self): self.create_exchange() self.bind_to_channel() + if not self.name: + self.name = self.channel.name + MSG_TYPES = ( diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 1327e3ad..d8c4bf42 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -254,7 +254,7 @@ def list_channels(current): } """ current.output['channels'] = [ - {'name': sbs.channel.name or ('Notifications' if sbs.channel.is_private() else ''), + {'name': sbs.name, 'key': sbs.channel.key, 'type': sbs.channel.typ, 'read_only': sbs.read_only, @@ -459,11 +459,13 @@ def create_direct_channel(current): 'status': 'Created', 'code': 201, 'channel_key': key, # of just created channel + 'name': string, # name of subscribed channel } """ - channel = Channel.get_or_create_direct_channel(current.user_id, current.input['user_key']) + channel, sub_name = Channel.get_or_create_direct_channel(current.user_id, current.input['user_key']) current.output = { 'channel_key': channel.key, + 'name': sub_name, 'status': 'OK', 'code': 201 } From 946cf4c6cdc19d71fa95ce556a4a9d3b0c018fd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Tue, 19 Jul 2016 16:55:40 +0300 Subject: [PATCH 35/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/lib.py | 6 ++---- zengine/messaging/model.py | 9 ++++++++- zengine/messaging/views.py | 1 + 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py index 94d4734d..7a58a753 100644 --- a/zengine/messaging/lib.py +++ b/zengine/messaging/lib.py @@ -65,11 +65,9 @@ def set_password(self, raw_password): def is_online(self, status=None): if status is None: - return ConnectionStatus(self.key).get() + return ConnectionStatus(self.key).get() or False ConnectionStatus(self.key).set(status) - if status == False: - pass - # TODO: do + def encrypt_password(self): """ encrypt password if not already encrypted """ diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 17002b1f..e3ba6542 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -88,7 +88,7 @@ def get_or_create_direct_channel(cls, initiator_key, receiver_key): return existing[0], receiver_name else: channel_name = '%s_%s' % (initiator_key, receiver_key) - channel = cls(is_direct=True, code_name=channel_name).save() + channel = cls(is_direct=True, code_name=channel_name, typ=10).save() Subscriber(channel=channel, user_id=initiator_key, name=receiver_name).save() @@ -189,6 +189,13 @@ def get_actions(self): ('Add Users', '_zops_add_members'), ('Add Unit', '_zops_add_unit_to_channel') ]) + return actions + + def is_online(self): + # TODO: Cache this method + if self.channel.typ == 10: + return self.channel.subscriber_set.objects.exclude(user=self.user).get().user.is_online() + def unread_count(self): # FIXME: track and return actual unread message count diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index d8c4bf42..b93e4c8a 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -258,6 +258,7 @@ def list_channels(current): 'key': sbs.channel.key, 'type': sbs.channel.typ, 'read_only': sbs.read_only, + 'is_online': sbs.is_online(), 'actions': sbs.get_actions(), 'unread': sbs.unread_count()} for sbs in current.user.subscriptions.objects.filter(is_visible=True)] From 86c5fa848483e4f173f0e0d568ec21a3e8163795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Tue, 19 Jul 2016 19:39:42 +0300 Subject: [PATCH 36/61] minor bug fixes --- zengine/messaging/model.py | 1 + zengine/messaging/views.py | 1 + 2 files changed, 2 insertions(+) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index e3ba6542..361a8db1 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -312,6 +312,7 @@ def serialize(self, user=None): 'title': self.msg_title, 'sender_name': self.sender.full_name, 'sender_key': self.sender.key, + 'avatar_url': self.sender.avatar, 'key': self.key, 'actions': self.get_actions_for(user), } diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index b93e4c8a..5d6ae1e9 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -30,6 +30,7 @@ 'sender_name': string, 'sender_key': key, 'type': int, + 'avatar_url': string, 'key': key, 'actions':[('action name', 'view name'), ('Add to Favorite', '_zops_add_to_favorites'), # applicable to everyone From 09da78bcb096d23af2cc0e362175f275f18baa55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Tue, 19 Jul 2016 19:53:42 +0300 Subject: [PATCH 37/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/views.py | 4 ++-- zengine/tornado_server/ws_to_queue.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 5d6ae1e9..390fd6a3 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -157,7 +157,7 @@ def show_channel(current): 'member_list': [{'name': sb.user.full_name, 'is_online': sb.user.is_online(), 'avatar_url': sb.user.get_avatar_url() - } for sb in ch.subscriber_set], + } for sb in ch.subscriber_set.objects.filter()], 'last_messages': [msg.serialize(current.user) for msg in ch.get_last_messages()] } @@ -411,7 +411,7 @@ def search_user(current): } for user in UserModel(current).objects.search_on(*settings.MESSAGING_USER_SEARCH_FIELDS, contains=current.input['query']): - current.output['results'].append((user.full_name, user.key, user.avatar)) + current.output['results'].append((user.full_name, user.key, user.get_avatar_url())) def search_unit(current): diff --git a/zengine/tornado_server/ws_to_queue.py b/zengine/tornado_server/ws_to_queue.py index cf8e9f4b..037333e3 100644 --- a/zengine/tornado_server/ws_to_queue.py +++ b/zengine/tornado_server/ws_to_queue.py @@ -268,13 +268,13 @@ def on_message(self, channel, method, header, body): if user_id in self.websockets: log.info("write msg to client") self.websockets[user_id].write_message(body) - channel.basic_ack(delivery_tag=method.delivery_tag) + # channel.basic_ack(delivery_tag=method.delivery_tag) elif 'sessid_to_userid' in body: reply = json_decode(body) sys.sessid_to_userid[reply['sess_id']] = reply['user_id'] self.websockets[reply['user_id']] = self.websockets[reply['sess_id']] del self.websockets[reply['sess_id']] - channel.basic_ack(delivery_tag=method.delivery_tag) + channel.basic_ack(delivery_tag=method.delivery_tag) # else: # channel.basic_reject(delivery_tag=method.delivery_tag) From 6928e29594e639060ad80fabf67663fe815392c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Wed, 20 Jul 2016 13:09:27 +0300 Subject: [PATCH 38/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- tests/async_amqp/messaging_tests.py | 50 +++++++++ zengine/lib/concurrent_amqp_test_client.py | 116 ++++++++++++++------- zengine/messaging/model.py | 26 +++-- zengine/messaging/views.py | 8 +- zengine/tornado_server/ws_to_queue.py | 4 +- 5 files changed, 148 insertions(+), 56 deletions(-) create mode 100644 tests/async_amqp/messaging_tests.py diff --git a/tests/async_amqp/messaging_tests.py b/tests/async_amqp/messaging_tests.py new file mode 100644 index 00000000..e17a6e78 --- /dev/null +++ b/tests/async_amqp/messaging_tests.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +""" +""" + +# Copyright (C) 2015 ZetaOps Inc. +# +# This file is licensed under the GNU General Public License v3 +# (GPLv3). See LICENSE.txt for details. +from zengine.lib.concurrent_amqp_test_client import ConcurrentTestCase, TestQueueManager + + +class TestCase(ConcurrentTestCase): + def test_channel_list(self): + self.post('ulakbus', {"view": "_zops_list_channels"}) + + def test_search_user(self): + self.post('ulakbus', {"view": "_zops_search_user", + "query": "x"}) + + def test_show_channel(self): + self.post('ulakbus', + {"view": "_zops_show_channel", + 'channel_key': 'iG4mvjQrfkvTDvM6Jk56X5ILoJ_CoqwpemOHnknn3hYu1BlAghb3dm'}) + + def test_create_message(self): + self.post('ulakbus', + {"view": "_zops_create_message", + "message": dict( + body='test_body', title='testtitle', + channel='iG4mvjQrfkvTDvM6Jk56X5ILoJ_CoqwpemOHnknn3hYu1BlAghb3dm', + receiver='', + type=2 + )}) + + +def main(): + from tornado import ioloop + # initiate amqp manager + ioloop = ioloop.IOLoop.instance() + qm = TestQueueManager(io_loop=ioloop) + + # initiate test case + qm.set_test_class(TestCase) + + qm.connect() + ioloop.start() + + +if __name__ == '__main__': + main() diff --git a/zengine/lib/concurrent_amqp_test_client.py b/zengine/lib/concurrent_amqp_test_client.py index 0406f1bf..df42fb0a 100644 --- a/zengine/lib/concurrent_amqp_test_client.py +++ b/zengine/lib/concurrent_amqp_test_client.py @@ -13,7 +13,11 @@ # # This file is licensed under the GNU General Public License v3 # (GPLv3). See LICENSE.txt for details. +from __future__ import print_function + +import inspect import uuid +from pprint import pprint import pika from tornado.escape import json_encode, json_decode @@ -29,6 +33,7 @@ from zengine.views.auth import Login sys.sessid_to_userid = {} +sys.test_method_names = {} UserModel = get_object_from_path(settings.USER_MODEL) @@ -70,7 +75,6 @@ def __init__(self, queue_manager, username, sess_id=None): self.request = type('MockWSRequestObject', (object,), {'remote_ip': '127.0.0.1'}) self.queue_manager = queue_manager - # order is important! self.sess_id = sess_id or uuid.uuid4().hex sys.sessid_to_userid[self.sess_id] = self.user.key.lower() self.queue_manager.register_websocket(self.sess_id, self) @@ -96,13 +100,17 @@ def backend_to_client(self, body): self.message_stack[body['callbackID']] = body log.info("WRITE MESSAGE TO CLIENT:\n%s" % (body,)) - def client_to_backend(self, message, callback): + def client_to_backend(self, message, callback, caller_fn_name): """ from client to backend """ cbid = uuid.uuid4().hex - self.message_callbacks[cbid] = callback message = json_encode({"callbackID": cbid, "data": message}) + def cb(res): + print("Testing: %s :: " % caller_fn_name, end='') + callback(res, message) + # self.message_callbacks[cbid] = lambda res: callable(res, message) + self.message_callbacks[cbid] = cb log.info("GOT MESSAGE FOR BACKEND %s: %s" % (self.sess_id, message)) self.queue_manager.redirect_incoming_message(self.sess_id, message, self.request) @@ -116,18 +124,14 @@ class ConcurrentTestCase(object): """ def __init__(self, queue_manager): - from tornado import ioloop log.info("ConcurrentTestCase class init with %s" % queue_manager) - ioloop = ioloop.IOLoop.instance() - self.ws1 = self.get_client('ulakbus') - self.ws2 = self.get_client('ogrenci_isleri_1') - self.queue_manager = TestQueueManager(io_loop=ioloop) - # initiate amqp manager - self.queue_manager.set_test_class(self.run_tests) - self.queue_manager.connect() - ioloop.start() - - def get_client(self, username): + self.queue_manager = queue_manager + self.clients = {} + self.make_client('ulakbus') + self.test_fn_name = '' + self.run_tests() + + def make_client(self, username): """ Args: username: username for this client instance @@ -135,28 +139,70 @@ def get_client(self, username): Returns: Logged in TestWSClient instance for given username """ - return TestWSClient(self.queue_manager, username) + self.clients[username] = TestWSClient(self.queue_manager, username) + + + + def post(self, username, data, callback=None): + if username not in self.clients: + self.make_client(username) + callback = callback or self.stc + self.clients[username].client_to_backend(data, callback, self.test_fn_name) def run_tests(self): for name in sorted(self.__class__.__dict__): if name.startswith("test_"): - try: - getattr(self, name)() - print("%s succesfully passed" % name) - except: - print("%s FAIL" % name) - - def success_test_callback(self, response, request=None): - # print(response) - assert response['code'] in (200, 201), "Process response not successful: \n %s \n %s" % ( - response, request - ) - - def test_channel_list(self): - self.ws1.client_to_backend({"view": "_zops_list_channels"}, - self.success_test_callback) - - def test_search_user(self): - self.ws1.client_to_backend({"view": "_zops_search_user", - "query": "x"}, - self.success_test_callback) + self.test_fn_name = name[5:] + getattr(self, name)() + + def process_error_reponse(self, resp): + if 'error' in resp: + print(resp['error'].replace('\\n','\n').replace('u\\', '')) + return True + + def stc(self, response, request=None): + """ + STC means Success Test Callback. Looks for 200 or 201 codes in response code. + + Args: + response: + request: + """ + if not response['code'] in (200, 201): + print("FAILED: Response not successful: \n") + if not self.process_error_reponse(response): + print("\nRESP:\n%s") + print("\nREQ:\n %s" % (response, request)) + else: + print("PASS!\n") + + def pstc(self, response, request=None): + """ + Same as self.stc() (success request callback) but printing response/request + for debugging purposes + + Args: + response: + request: + + """ + self.stc(response, request) + print("\n\n=================\n\nRESPONSE: %s \n\nREQUEST: %s\n" % (response, request)) + + + +def main(): + from tornado import ioloop + # initiate amqp manager + ioloop = ioloop.IOLoop.instance() + qm = TestQueueManager(io_loop=ioloop) + + # initiate test case + qm.set_test_class(ConcurrentTestCase) + + qm.connect() + ioloop.start() + + +if __name__ == '__main__': + main() diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 361a8db1..6d4483f8 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -14,6 +14,7 @@ from pyoko import Model, field, ListNode from pyoko.conf import settings from pyoko.exceptions import IntegrityError +from pyoko.fields import DATE_TIME_FORMAT from pyoko.lib.utils import get_object_from_path from zengine.client_queue import BLOCKING_MQ_PARAMS from zengine.lib.utils import to_safe_str @@ -276,18 +277,15 @@ class Message(Model): url = field.String("URL") def get_actions_for(self, user): - actions = [ - ('Favorite', '_zops_favorite_message') - ] - if self.sender == user: - actions.extend([ - ('Delete', '_zops_delete_message'), - ('Edit', '_zops_edit_message') - ]) - elif user: - actions.extend([ - ('Flag', '_zops_flag_message') - ]) + actions = [('Favorite', '_zops_favorite_message')] + if user: + actions.extend([('Flag', '_zops_flag_message')]) + if self.sender == user: + actions.extend([ + ('Delete', '_zops_delete_message'), + ('Edit', '_zops_edit_message') + ]) + def serialize(self, user=None): """ @@ -305,8 +303,8 @@ def serialize(self, user=None): return { 'content': self.body, 'type': self.typ, - 'updated_at': self.updated_at, - 'timestamp': self.timestamp, + 'updated_at': self.updated_at.strftime(DATE_TIME_FORMAT) if self.updated_at else None, + 'timestamp': self.timestamp.strftime(DATE_TIME_FORMAT), 'is_update': self.exist, 'attachments': [attachment.serialize() for attachment in self.attachment_set], 'title': self.msg_title, diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 390fd6a3..71a8a5ec 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -110,15 +110,13 @@ def create_message(current): """ msg = current.input['message'] - ch = Channel(current).objects.get(msg['channel']) - msg_obj = ch.add_message(body=msg['body'], typ=msg['typ'], sender=current.user, + msg_obj = Channel.add_message(msg['channel'], body=msg['body'], typ=msg['type'], sender=current.user, title=msg['title'], receiver=msg['receiver'] or None) if 'attachment' in msg: for atch in msg['attachments']: - # TODO: Attachment type detection typ = current._dedect_file_type(atch['name'], atch['content']) - Attachment(channel=ch, msg=msg_obj, name=atch['name'], file=atch['content'], - description=atch['description'], typ=typ).save() + Attachment(channel_id=msg['channel'], msg=msg_obj, name=atch['name'], + file=atch['content'], description=atch['description'], typ=typ).save() def show_channel(current): diff --git a/zengine/tornado_server/ws_to_queue.py b/zengine/tornado_server/ws_to_queue.py index 037333e3..cf8e9f4b 100644 --- a/zengine/tornado_server/ws_to_queue.py +++ b/zengine/tornado_server/ws_to_queue.py @@ -268,13 +268,13 @@ def on_message(self, channel, method, header, body): if user_id in self.websockets: log.info("write msg to client") self.websockets[user_id].write_message(body) - # channel.basic_ack(delivery_tag=method.delivery_tag) + channel.basic_ack(delivery_tag=method.delivery_tag) elif 'sessid_to_userid' in body: reply = json_decode(body) sys.sessid_to_userid[reply['sess_id']] = reply['user_id'] self.websockets[reply['user_id']] = self.websockets[reply['sess_id']] del self.websockets[reply['sess_id']] - channel.basic_ack(delivery_tag=method.delivery_tag) + channel.basic_ack(delivery_tag=method.delivery_tag) # else: # channel.basic_reject(delivery_tag=method.delivery_tag) From 4b9e238aa844e4964820118e82c33d5b464330d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Thu, 21 Jul 2016 07:28:20 +0300 Subject: [PATCH 39/61] pre-e2e-refactor rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- tests/async_amqp/messaging_tests.py | 3 +-- zengine/lib/concurrent_amqp_test_client.py | 15 ++++++++------- zengine/messaging/model.py | 8 +++++--- zengine/messaging/views.py | 6 ++++++ zengine/tornado_server/ws_to_queue.py | 4 ++-- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/async_amqp/messaging_tests.py b/tests/async_amqp/messaging_tests.py index e17a6e78..254b4ad4 100644 --- a/tests/async_amqp/messaging_tests.py +++ b/tests/async_amqp/messaging_tests.py @@ -14,8 +14,7 @@ def test_channel_list(self): self.post('ulakbus', {"view": "_zops_list_channels"}) def test_search_user(self): - self.post('ulakbus', {"view": "_zops_search_user", - "query": "x"}) + self.post('ulakbus', {"view": "_zops_search_user", "query": "x"}) def test_show_channel(self): self.post('ulakbus', diff --git a/zengine/lib/concurrent_amqp_test_client.py b/zengine/lib/concurrent_amqp_test_client.py index df42fb0a..ffb7fdda 100644 --- a/zengine/lib/concurrent_amqp_test_client.py +++ b/zengine/lib/concurrent_amqp_test_client.py @@ -33,7 +33,6 @@ from zengine.views.auth import Login sys.sessid_to_userid = {} -sys.test_method_names = {} UserModel = get_object_from_path(settings.USER_MODEL) @@ -97,6 +96,8 @@ def backend_to_client(self, body): try: self.message_callbacks[body['callbackID']](body) except KeyError: + print("No cb for %s" % body['callbackID']) + print("CB HELL %s" % self.message_callbacks) self.message_stack[body['callbackID']] = body log.info("WRITE MESSAGE TO CLIENT:\n%s" % (body,)) @@ -107,10 +108,11 @@ def client_to_backend(self, message, callback, caller_fn_name): cbid = uuid.uuid4().hex message = json_encode({"callbackID": cbid, "data": message}) def cb(res): - print("Testing: %s :: " % caller_fn_name, end='') - callback(res, message) + result = callback(res, message) + print("API Request: %s :: %s\n" % (caller_fn_name, 'PASS' if result else 'FAIL!')) # self.message_callbacks[cbid] = lambda res: callable(res, message) self.message_callbacks[cbid] = cb + print(caller_fn_name, self.message_callbacks) log.info("GOT MESSAGE FOR BACKEND %s: %s" % (self.sess_id, message)) self.queue_manager.redirect_incoming_message(self.sess_id, message, self.request) @@ -128,7 +130,6 @@ def __init__(self, queue_manager): self.queue_manager = queue_manager self.clients = {} self.make_client('ulakbus') - self.test_fn_name = '' self.run_tests() def make_client(self, username): @@ -147,12 +148,12 @@ def post(self, username, data, callback=None): if username not in self.clients: self.make_client(username) callback = callback or self.stc - self.clients[username].client_to_backend(data, callback, self.test_fn_name) + view_name = data['view'] if 'view' in data else sys._getframe(1).f_code.co_name + self.clients[username].client_to_backend(data, callback, view_name) def run_tests(self): for name in sorted(self.__class__.__dict__): if name.startswith("test_"): - self.test_fn_name = name[5:] getattr(self, name)() def process_error_reponse(self, resp): @@ -174,7 +175,7 @@ def stc(self, response, request=None): print("\nRESP:\n%s") print("\nREQ:\n %s" % (response, request)) else: - print("PASS!\n") + return True def pstc(self, response, request=None): """ diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 6d4483f8..394f12aa 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -228,8 +228,9 @@ def bind_to_channel(self): Binds (subscribes) users private exchange to channel exchange Automatically called at creation of subscription record. """ - channel = self._connect_mq() - channel.exchange_bind(source=self.channel.code_name, destination=self.user.prv_exchange) + if self.channel.code_name != self.user.prv_exchange: + channel = self._connect_mq() + channel.exchange_bind(source=self.channel.code_name, destination=self.user.prv_exchange) def post_creation(self): self.create_exchange() @@ -303,13 +304,14 @@ def serialize(self, user=None): return { 'content': self.body, 'type': self.typ, - 'updated_at': self.updated_at.strftime(DATE_TIME_FORMAT) if self.updated_at else None, + 'updated_at': self.updated_at, 'timestamp': self.timestamp.strftime(DATE_TIME_FORMAT), 'is_update': self.exist, 'attachments': [attachment.serialize() for attachment in self.attachment_set], 'title': self.msg_title, 'sender_name': self.sender.full_name, 'sender_key': self.sender.key, + 'cmd': 'message', 'avatar_url': self.sender.avatar, 'key': self.key, 'actions': self.get_actions_for(user), diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 71a8a5ec..26f14d44 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -32,6 +32,7 @@ 'type': int, 'avatar_url': string, 'key': key, + 'cmd': 'message', 'actions':[('action name', 'view name'), ('Add to Favorite', '_zops_add_to_favorites'), # applicable to everyone @@ -112,6 +113,11 @@ def create_message(current): msg = current.input['message'] msg_obj = Channel.add_message(msg['channel'], body=msg['body'], typ=msg['type'], sender=current.user, title=msg['title'], receiver=msg['receiver'] or None) + current.output = { + 'msg_key': msg_obj.key, + 'status': 'OK', + 'code': 201 + } if 'attachment' in msg: for atch in msg['attachments']: typ = current._dedect_file_type(atch['name'], atch['content']) diff --git a/zengine/tornado_server/ws_to_queue.py b/zengine/tornado_server/ws_to_queue.py index cf8e9f4b..037333e3 100644 --- a/zengine/tornado_server/ws_to_queue.py +++ b/zengine/tornado_server/ws_to_queue.py @@ -268,13 +268,13 @@ def on_message(self, channel, method, header, body): if user_id in self.websockets: log.info("write msg to client") self.websockets[user_id].write_message(body) - channel.basic_ack(delivery_tag=method.delivery_tag) + # channel.basic_ack(delivery_tag=method.delivery_tag) elif 'sessid_to_userid' in body: reply = json_decode(body) sys.sessid_to_userid[reply['sess_id']] = reply['user_id'] self.websockets[reply['user_id']] = self.websockets[reply['sess_id']] del self.websockets[reply['sess_id']] - channel.basic_ack(delivery_tag=method.delivery_tag) + channel.basic_ack(delivery_tag=method.delivery_tag) # else: # channel.basic_reject(delivery_tag=method.delivery_tag) From 82de442c8fb3e048dd66c892014c80c1032fbdb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Thu, 21 Jul 2016 10:51:57 +0300 Subject: [PATCH 40/61] e2e-removal-refactor finished. instant messaging working rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/lib.py | 8 ++-- zengine/messaging/model.py | 43 ++++++++++----------- zengine/models/auth.py | 2 +- zengine/tornado_server/ws_to_queue.py | 54 ++++++++------------------- zengine/views/auth.py | 11 +----- zengine/views/system.py | 2 +- zengine/wf_daemon.py | 8 ++-- 7 files changed, 47 insertions(+), 81 deletions(-) diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py index 7a58a753..27057e77 100644 --- a/zengine/messaging/lib.py +++ b/zengine/messaging/lib.py @@ -74,7 +74,8 @@ def encrypt_password(self): if self.password and not self.password.startswith('$pbkdf2'): self.set_password(self.password) - def prepare_channels(self): + def prepare_user_channel(self): + """should be called from User.post_creation hook""" from zengine.messaging.model import Channel, Subscriber # create private channel of user ch, new = Channel.objects.get_or_create(owner=self, typ=5) @@ -117,12 +118,13 @@ def full_name(self): def prv_exchange(self): return 'prv_%s' % str(self.key).lower() - def bind_private_channel(self, sess_id): + def bind_channels_to_session_queue(self, sess_id): mq_channel = pika.BlockingConnection(BLOCKING_MQ_PARAMS).channel() mq_channel.queue_declare(queue=sess_id, arguments={'x-expires': 40000}) log.debug("Binding private exchange to client queue: Q:%s --> E:%s" % (sess_id, self.prv_exchange)) - mq_channel.queue_bind(exchange=self.prv_exchange, queue=sess_id) + for sbs in self.subscriptions.objects.filter(): + mq_channel.queue_bind(exchange=sbs.channel.code_name, queue=sess_id) def send_notification(self, title, message, typ=1, url=None): """ diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 394f12aa..52a0ef54 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -17,8 +17,9 @@ from pyoko.fields import DATE_TIME_FORMAT from pyoko.lib.utils import get_object_from_path from zengine.client_queue import BLOCKING_MQ_PARAMS +from zengine.lib.cache import UserSessionID from zengine.lib.utils import to_safe_str - +from zengine.log import log UserModel = get_object_from_path(settings.USER_MODEL) @@ -105,7 +106,7 @@ def add_message(cls, channel_key, body, title=None, sender=None, url=None, typ=2 msg_object = Message(sender=sender, body=body, msg_title=title, url=url, typ=typ, channel_id=channel_key, receiver=receiver, key=uuid4().hex) mq_channel.basic_publish(exchange=channel_key, - routing_key='', + routing_key='#', body=json.dumps(msg_object.serialize())) return msg_object.save() @@ -205,35 +206,29 @@ def unread_count(self): else: self.channel.message_set.objects.filter().count() - def create_exchange(self): - """ - Creates user's private exchange - - Actually user's private channel needed to be defined only once, - and this should be happened when user first created. - But since this has a little performance cost, - to be safe we always call it before binding to the channel we currently subscribe - """ - channel = self._connect_mq() - channel.exchange_declare(exchange='prv_%s' % self.user.key.lower(), - exchange_type='fanout', - durable=True) - @classmethod def mark_seen(cls, key, datetime_str): cls.objects.filter(key=key).update(last_seen=datetime_str) def bind_to_channel(self): """ - Binds (subscribes) users private exchange to channel exchange + Binds (subscribes) users session queue to channel exchange Automatically called at creation of subscription record. + Same operation done at user login + (zengine.messaging.lib.BaseUser#bind_channels_to_session_queue) """ - if self.channel.code_name != self.user.prv_exchange: - channel = self._connect_mq() - channel.exchange_bind(source=self.channel.code_name, destination=self.user.prv_exchange) + try: + sess_id = UserSessionID(self.user_id).get() + mq_channel = self._connect_mq() + mq_channel.queue_bind(exchange=self.channel.code_name, + queue=sess_id) + except: + log.exception("Cant create subscription binding for %s : %s" % (self.name, + self.user.full_name)) + def post_creation(self): - self.create_exchange() + # self.create_exchange() self.bind_to_channel() if not self.name: @@ -329,9 +324,9 @@ def _republish(self): mq_channel.basic_publish(exchange=self.channel.key, routing_key='', body=json.dumps(self.serialize())) - def pre_save(self): - if self.exist: - self._republish() + # def pre_save(self): + # if self.exist: + # self._republish() ATTACHMENT_TYPES = ( diff --git a/zengine/models/auth.py b/zengine/models/auth.py index 2df93ed7..79a06767 100644 --- a/zengine/models/auth.py +++ b/zengine/models/auth.py @@ -89,7 +89,7 @@ def pre_save(self): self.encrypt_password() def post_creation(self): - self.prepare_channels() + self.prepare_user_channel() def get_permissions(self): """ diff --git a/zengine/tornado_server/ws_to_queue.py b/zengine/tornado_server/ws_to_queue.py index 037333e3..8850c1d5 100644 --- a/zengine/tornado_server/ws_to_queue.py +++ b/zengine/tornado_server/ws_to_queue.py @@ -118,6 +118,7 @@ def send_message(self, sess_id, input_data): {'code': 503, 'error': 'Retry'}) + class QueueManager(object): """ Async RabbitMQ & Tornado websocket connector @@ -134,7 +135,7 @@ def __init__(self, io_loop=None): self.out_channels = {} self.out_channel = None self.websockets = {} - # self.connect() + def connect(self): """ @@ -187,12 +188,6 @@ def on_input_queue_declare(self, queue): exchange='input_exc', queue=self.INPUT_QUEUE_NAME, routing_key="#") - def ask_for_user_id(self, sess_id): - log.debug(sess_id) - # TODO: add remote ip - self.publish_incoming_message(dict(_zops_remote_ip='', - data={'view': 'sessid_to_userid'}), sess_id) - def register_websocket(self, sess_id, ws): """ @@ -201,15 +196,8 @@ def register_websocket(self, sess_id, ws): sess_id: ws: """ - log.debug("GET SESSUSERS: %s" % sys.sessid_to_userid) - try: - user_id = sys.sessid_to_userid[sess_id] - self.websockets[user_id] = ws - except KeyError: - self.ask_for_user_id(sess_id) - self.websockets[sess_id] = ws - user_id = sess_id - self.create_out_channel(sess_id, user_id) + self.websockets[sess_id] = ws + self.create_out_channel(sess_id) def inform_disconnection(self, sess_id): self.in_channel.basic_publish(exchange='input_exc', @@ -220,25 +208,22 @@ def inform_disconnection(self, sess_id): _zops_remote_ip=''))) def unregister_websocket(self, sess_id): - user_id = sys.sessid_to_userid.get(sess_id, None) try: self.inform_disconnection(sess_id) - del self.websockets[user_id] - except KeyError: - log.exception("Non-existent websocket for %s" % user_id) + del self.websockets[sess_id] + except: + log.exception("WS already deleted") if sess_id in self.out_channels: try: self.out_channels[sess_id].close() except ChannelClosed: log.exception("Pika client (out) channel already closed") - def create_out_channel(self, sess_id, user_id): + def create_out_channel(self, sess_id): def _on_output_channel_creation(channel): def _on_output_queue_decleration(queue): - channel.basic_consume(self.on_message, queue=sess_id) - log.debug("BIND QUEUE TO WS Q.%s on Ch.%s WS.%s" % (sess_id, - channel.consumer_tags[0], - user_id)) + channel.basic_consume(self.on_message, queue=sess_id, consumer_tag=sess_id) + log.debug("BIND QUEUE TO WS Q.%s " % sess_id) self.out_channels[sess_id] = channel channel.queue_declare(callback=_on_output_queue_decleration, @@ -263,18 +248,9 @@ def publish_incoming_message(self, message, sess_id): body=json_encode(message)) def on_message(self, channel, method, header, body): - user_id = method.exchange[4:] - log.debug("WS RPLY for %s: %s" % (user_id, body)) - if user_id in self.websockets: - log.info("write msg to client") - self.websockets[user_id].write_message(body) - # channel.basic_ack(delivery_tag=method.delivery_tag) - elif 'sessid_to_userid' in body: - reply = json_decode(body) - sys.sessid_to_userid[reply['sess_id']] = reply['user_id'] - self.websockets[reply['user_id']] = self.websockets[reply['sess_id']] - del self.websockets[reply['sess_id']] + sess_id = method.consumer_tag + log.debug("WS RPLY for %s: %s" % (sess_id, body)) + if sess_id in self.websockets: + log.info("write msg to client %s" % sess_id) + self.websockets[sess_id].write_message(body) channel.basic_ack(delivery_tag=method.delivery_tag) - - # else: - # channel.basic_reject(delivery_tag=method.delivery_tag) diff --git a/zengine/views/auth.py b/zengine/views/auth.py index d8571fdf..62be4f88 100644 --- a/zengine/views/auth.py +++ b/zengine/views/auth.py @@ -61,9 +61,8 @@ def _do_upgrade(self): self.current.output['cmd'] = 'upgrade' self.current.output['user_id'] = self.current.user_id self.current.user.is_online(True) - self.current.user.bind_private_channel(self.current.session.sess_id) - user_sess = UserSessionID(self.current.user_id) - user_sess.set(self.current.session.sess_id) + self.current.user.bind_channels_to_session_queue(self.current.session.sess_id) + UserSessionID(self.current.user_id).set(self.current.session.sess_id) def do_view(self): """ @@ -82,12 +81,6 @@ def do_view(self): self.current.task_data['login_successful'] = auth_result if auth_result: self._do_upgrade() - - # old_sess_id = user_sess.get() - # notify = Notify(self.current.user_id) - # notify.cache_to_queue() - # if old_sess_id: - # notify.old_to_new_queue(old_sess_id) except: raise self.current.log.exception("Wrong username or another error occurred") diff --git a/zengine/views/system.py b/zengine/views/system.py index a592faf2..a82ff941 100644 --- a/zengine/views/system.py +++ b/zengine/views/system.py @@ -11,7 +11,7 @@ def sessid_to_userid(current): current.output['user_id'] = current.user_id.lower() current.output['sess_id'] = current.session.sess_id - current.user.bind_private_channel(current.session.sess_id) + current.user.bind_channels_to_session_queue(current.session.sess_id) current.output['sessid_to_userid'] = True def mark_offline_user(current): diff --git a/zengine/wf_daemon.py b/zengine/wf_daemon.py index 4649ac15..0d6fdfd9 100755 --- a/zengine/wf_daemon.py +++ b/zengine/wf_daemon.py @@ -176,10 +176,10 @@ def handle_message(self, ch, method, properties, body): def send_output(self, output): # TODO: This is ugly, we should separate login process log.debug("SEND_OUTPUT: %s" % output) - if self.current.user_id is None or 'login_process' in output: - self.client_queue.send_to_default_exchange(self.sessid, output) - else: - self.client_queue.send_to_prv_exchange(self.current.user_id, output) + # if self.current.user_id is None or 'login_process' in output: + self.client_queue.send_to_default_exchange(self.sessid, output) + # else: + # self.client_queue.send_to_prv_exchange(self.current.user_id, output) From 4b67472862b240c2508ced1a077a21b5cb0e27fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Thu, 21 Jul 2016 12:27:46 +0300 Subject: [PATCH 41/61] Revert "e2e-removal-refactor finished. instant messaging working" This reverts commit 82de442c8fb3e048dd66c892014c80c1032fbdb2. --- zengine/messaging/lib.py | 8 ++-- zengine/messaging/model.py | 43 +++++++++++---------- zengine/models/auth.py | 2 +- zengine/tornado_server/ws_to_queue.py | 54 +++++++++++++++++++-------- zengine/views/auth.py | 11 +++++- zengine/views/system.py | 2 +- zengine/wf_daemon.py | 8 ++-- 7 files changed, 81 insertions(+), 47 deletions(-) diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py index 27057e77..7a58a753 100644 --- a/zengine/messaging/lib.py +++ b/zengine/messaging/lib.py @@ -74,8 +74,7 @@ def encrypt_password(self): if self.password and not self.password.startswith('$pbkdf2'): self.set_password(self.password) - def prepare_user_channel(self): - """should be called from User.post_creation hook""" + def prepare_channels(self): from zengine.messaging.model import Channel, Subscriber # create private channel of user ch, new = Channel.objects.get_or_create(owner=self, typ=5) @@ -118,13 +117,12 @@ def full_name(self): def prv_exchange(self): return 'prv_%s' % str(self.key).lower() - def bind_channels_to_session_queue(self, sess_id): + def bind_private_channel(self, sess_id): mq_channel = pika.BlockingConnection(BLOCKING_MQ_PARAMS).channel() mq_channel.queue_declare(queue=sess_id, arguments={'x-expires': 40000}) log.debug("Binding private exchange to client queue: Q:%s --> E:%s" % (sess_id, self.prv_exchange)) - for sbs in self.subscriptions.objects.filter(): - mq_channel.queue_bind(exchange=sbs.channel.code_name, queue=sess_id) + mq_channel.queue_bind(exchange=self.prv_exchange, queue=sess_id) def send_notification(self, title, message, typ=1, url=None): """ diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 52a0ef54..394f12aa 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -17,9 +17,8 @@ from pyoko.fields import DATE_TIME_FORMAT from pyoko.lib.utils import get_object_from_path from zengine.client_queue import BLOCKING_MQ_PARAMS -from zengine.lib.cache import UserSessionID from zengine.lib.utils import to_safe_str -from zengine.log import log + UserModel = get_object_from_path(settings.USER_MODEL) @@ -106,7 +105,7 @@ def add_message(cls, channel_key, body, title=None, sender=None, url=None, typ=2 msg_object = Message(sender=sender, body=body, msg_title=title, url=url, typ=typ, channel_id=channel_key, receiver=receiver, key=uuid4().hex) mq_channel.basic_publish(exchange=channel_key, - routing_key='#', + routing_key='', body=json.dumps(msg_object.serialize())) return msg_object.save() @@ -206,29 +205,35 @@ def unread_count(self): else: self.channel.message_set.objects.filter().count() + def create_exchange(self): + """ + Creates user's private exchange + + Actually user's private channel needed to be defined only once, + and this should be happened when user first created. + But since this has a little performance cost, + to be safe we always call it before binding to the channel we currently subscribe + """ + channel = self._connect_mq() + channel.exchange_declare(exchange='prv_%s' % self.user.key.lower(), + exchange_type='fanout', + durable=True) + @classmethod def mark_seen(cls, key, datetime_str): cls.objects.filter(key=key).update(last_seen=datetime_str) def bind_to_channel(self): """ - Binds (subscribes) users session queue to channel exchange + Binds (subscribes) users private exchange to channel exchange Automatically called at creation of subscription record. - Same operation done at user login - (zengine.messaging.lib.BaseUser#bind_channels_to_session_queue) """ - try: - sess_id = UserSessionID(self.user_id).get() - mq_channel = self._connect_mq() - mq_channel.queue_bind(exchange=self.channel.code_name, - queue=sess_id) - except: - log.exception("Cant create subscription binding for %s : %s" % (self.name, - self.user.full_name)) - + if self.channel.code_name != self.user.prv_exchange: + channel = self._connect_mq() + channel.exchange_bind(source=self.channel.code_name, destination=self.user.prv_exchange) def post_creation(self): - # self.create_exchange() + self.create_exchange() self.bind_to_channel() if not self.name: @@ -324,9 +329,9 @@ def _republish(self): mq_channel.basic_publish(exchange=self.channel.key, routing_key='', body=json.dumps(self.serialize())) - # def pre_save(self): - # if self.exist: - # self._republish() + def pre_save(self): + if self.exist: + self._republish() ATTACHMENT_TYPES = ( diff --git a/zengine/models/auth.py b/zengine/models/auth.py index 79a06767..2df93ed7 100644 --- a/zengine/models/auth.py +++ b/zengine/models/auth.py @@ -89,7 +89,7 @@ def pre_save(self): self.encrypt_password() def post_creation(self): - self.prepare_user_channel() + self.prepare_channels() def get_permissions(self): """ diff --git a/zengine/tornado_server/ws_to_queue.py b/zengine/tornado_server/ws_to_queue.py index 8850c1d5..037333e3 100644 --- a/zengine/tornado_server/ws_to_queue.py +++ b/zengine/tornado_server/ws_to_queue.py @@ -118,7 +118,6 @@ def send_message(self, sess_id, input_data): {'code': 503, 'error': 'Retry'}) - class QueueManager(object): """ Async RabbitMQ & Tornado websocket connector @@ -135,7 +134,7 @@ def __init__(self, io_loop=None): self.out_channels = {} self.out_channel = None self.websockets = {} - + # self.connect() def connect(self): """ @@ -188,6 +187,12 @@ def on_input_queue_declare(self, queue): exchange='input_exc', queue=self.INPUT_QUEUE_NAME, routing_key="#") + def ask_for_user_id(self, sess_id): + log.debug(sess_id) + # TODO: add remote ip + self.publish_incoming_message(dict(_zops_remote_ip='', + data={'view': 'sessid_to_userid'}), sess_id) + def register_websocket(self, sess_id, ws): """ @@ -196,8 +201,15 @@ def register_websocket(self, sess_id, ws): sess_id: ws: """ - self.websockets[sess_id] = ws - self.create_out_channel(sess_id) + log.debug("GET SESSUSERS: %s" % sys.sessid_to_userid) + try: + user_id = sys.sessid_to_userid[sess_id] + self.websockets[user_id] = ws + except KeyError: + self.ask_for_user_id(sess_id) + self.websockets[sess_id] = ws + user_id = sess_id + self.create_out_channel(sess_id, user_id) def inform_disconnection(self, sess_id): self.in_channel.basic_publish(exchange='input_exc', @@ -208,22 +220,25 @@ def inform_disconnection(self, sess_id): _zops_remote_ip=''))) def unregister_websocket(self, sess_id): + user_id = sys.sessid_to_userid.get(sess_id, None) try: self.inform_disconnection(sess_id) - del self.websockets[sess_id] - except: - log.exception("WS already deleted") + del self.websockets[user_id] + except KeyError: + log.exception("Non-existent websocket for %s" % user_id) if sess_id in self.out_channels: try: self.out_channels[sess_id].close() except ChannelClosed: log.exception("Pika client (out) channel already closed") - def create_out_channel(self, sess_id): + def create_out_channel(self, sess_id, user_id): def _on_output_channel_creation(channel): def _on_output_queue_decleration(queue): - channel.basic_consume(self.on_message, queue=sess_id, consumer_tag=sess_id) - log.debug("BIND QUEUE TO WS Q.%s " % sess_id) + channel.basic_consume(self.on_message, queue=sess_id) + log.debug("BIND QUEUE TO WS Q.%s on Ch.%s WS.%s" % (sess_id, + channel.consumer_tags[0], + user_id)) self.out_channels[sess_id] = channel channel.queue_declare(callback=_on_output_queue_decleration, @@ -248,9 +263,18 @@ def publish_incoming_message(self, message, sess_id): body=json_encode(message)) def on_message(self, channel, method, header, body): - sess_id = method.consumer_tag - log.debug("WS RPLY for %s: %s" % (sess_id, body)) - if sess_id in self.websockets: - log.info("write msg to client %s" % sess_id) - self.websockets[sess_id].write_message(body) + user_id = method.exchange[4:] + log.debug("WS RPLY for %s: %s" % (user_id, body)) + if user_id in self.websockets: + log.info("write msg to client") + self.websockets[user_id].write_message(body) + # channel.basic_ack(delivery_tag=method.delivery_tag) + elif 'sessid_to_userid' in body: + reply = json_decode(body) + sys.sessid_to_userid[reply['sess_id']] = reply['user_id'] + self.websockets[reply['user_id']] = self.websockets[reply['sess_id']] + del self.websockets[reply['sess_id']] channel.basic_ack(delivery_tag=method.delivery_tag) + + # else: + # channel.basic_reject(delivery_tag=method.delivery_tag) diff --git a/zengine/views/auth.py b/zengine/views/auth.py index 62be4f88..d8571fdf 100644 --- a/zengine/views/auth.py +++ b/zengine/views/auth.py @@ -61,8 +61,9 @@ def _do_upgrade(self): self.current.output['cmd'] = 'upgrade' self.current.output['user_id'] = self.current.user_id self.current.user.is_online(True) - self.current.user.bind_channels_to_session_queue(self.current.session.sess_id) - UserSessionID(self.current.user_id).set(self.current.session.sess_id) + self.current.user.bind_private_channel(self.current.session.sess_id) + user_sess = UserSessionID(self.current.user_id) + user_sess.set(self.current.session.sess_id) def do_view(self): """ @@ -81,6 +82,12 @@ def do_view(self): self.current.task_data['login_successful'] = auth_result if auth_result: self._do_upgrade() + + # old_sess_id = user_sess.get() + # notify = Notify(self.current.user_id) + # notify.cache_to_queue() + # if old_sess_id: + # notify.old_to_new_queue(old_sess_id) except: raise self.current.log.exception("Wrong username or another error occurred") diff --git a/zengine/views/system.py b/zengine/views/system.py index a82ff941..a592faf2 100644 --- a/zengine/views/system.py +++ b/zengine/views/system.py @@ -11,7 +11,7 @@ def sessid_to_userid(current): current.output['user_id'] = current.user_id.lower() current.output['sess_id'] = current.session.sess_id - current.user.bind_channels_to_session_queue(current.session.sess_id) + current.user.bind_private_channel(current.session.sess_id) current.output['sessid_to_userid'] = True def mark_offline_user(current): diff --git a/zengine/wf_daemon.py b/zengine/wf_daemon.py index 0d6fdfd9..4649ac15 100755 --- a/zengine/wf_daemon.py +++ b/zengine/wf_daemon.py @@ -176,10 +176,10 @@ def handle_message(self, ch, method, properties, body): def send_output(self, output): # TODO: This is ugly, we should separate login process log.debug("SEND_OUTPUT: %s" % output) - # if self.current.user_id is None or 'login_process' in output: - self.client_queue.send_to_default_exchange(self.sessid, output) - # else: - # self.client_queue.send_to_prv_exchange(self.current.user_id, output) + if self.current.user_id is None or 'login_process' in output: + self.client_queue.send_to_default_exchange(self.sessid, output) + else: + self.client_queue.send_to_prv_exchange(self.current.user_id, output) From 5f34aa7b0e78628955b2acec5346ed9b4d2c035c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Thu, 21 Jul 2016 16:43:07 +0300 Subject: [PATCH 42/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/management_commands.py | 5 ++- zengine/messaging/model.py | 48 ++++++++++++++++----------- zengine/messaging/views.py | 6 +++- zengine/tornado_server/ws_to_queue.py | 4 +-- zengine/wf_daemon.py | 2 +- 5 files changed, 41 insertions(+), 24 deletions(-) diff --git a/zengine/management_commands.py b/zengine/management_commands.py index 9229987f..fca3a793 100644 --- a/zengine/management_commands.py +++ b/zengine/management_commands.py @@ -210,7 +210,10 @@ def create_user_channels(self): sb, new = Subscriber.objects.get_or_create(channel=ch, user=usr, read_only=True, - name='Notifications') + name='Notifications', + can_manage=True, + can_leave=False + ) print("%s notify sub: %s" % ('created' if new else 'existing', ch.code_name)) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 394f12aa..c3ef6844 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -60,7 +60,9 @@ class Channel(Model): owner = UserModel(reverse_name='created_channels', null=True) def __unicode__(self): - return "%s (%s's %s channel)" % (self.name or '', self.owner.__unicode__(), self.get_typ_display()) + return "%s (%s's %s channel)" % (self.name or '', + self.owner.full_name, + self.get_typ_display()) # # class Managers(ListNode): @@ -86,17 +88,17 @@ def get_or_create_direct_channel(cls, initiator_key, receiver_key): code_name='%s_%s' % (receiver_key, initiator_key)) receiver_name = UserModel.objects.get(receiver_key).full_name if existing: - return existing[0], receiver_name + channel = existing[0] else: channel_name = '%s_%s' % (initiator_key, receiver_key) channel = cls(is_direct=True, code_name=channel_name, typ=10).save() - Subscriber(channel=channel, - user_id=initiator_key, - name=receiver_name).save() - Subscriber(channel=channel, - user_id=receiver_key, - name=UserModel.objects.get(initiator_key).full_name).save() - return channel, receiver_name + Subscriber.objects.get_or_create(channel=channel, + user_id=initiator_key, + name=receiver_name) + Subscriber.objects.get_or_create(channel=channel, + user_id=receiver_key, + name=UserModel.objects.get(initiator_key).full_name) + return channel, receiver_name @classmethod def add_message(cls, channel_key, body, title=None, sender=None, url=None, typ=2, @@ -104,6 +106,7 @@ def add_message(cls, channel_key, body, title=None, sender=None, url=None, typ=2 mq_channel = cls._connect_mq() msg_object = Message(sender=sender, body=body, msg_title=title, url=url, typ=typ, channel_id=channel_key, receiver=receiver, key=uuid4().hex) + msg_object.setattr('unsaved', True) mq_channel.basic_publish(exchange=channel_key, routing_key='', body=json.dumps(msg_object.serialize())) @@ -111,7 +114,7 @@ def add_message(cls, channel_key, body, title=None, sender=None, url=None, typ=2 def get_last_messages(self): # TODO: Try to refactor this with https://github.com/rabbitmq/rabbitmq-recent-history-exchange - return self.message_set.objects.filter()[:20] + return self.message_set.objects.filter().set_params(sort="timestamp asc")[:20] @classmethod def _connect_mq(cls): @@ -129,6 +132,12 @@ def create_exchange(self): exchange_type='fanout', durable=True) + def subscribe_owner(self): + sbs, new = Subscriber.objects.get_or_create(user=self.owner, + channel=self, + can_manage=True, + can_leave=False) + def pre_creation(self): if not self.code_name: if self.name: @@ -145,6 +154,7 @@ def pre_creation(self): def post_save(self): self.create_exchange() + # self.subscribe_owner() class Subscriber(Model): @@ -171,7 +181,7 @@ class Subscriber(Model): # status = field.Integer("Status", choices=SUBSCRIPTION_STATUS) def __unicode__(self): - return "%s >> %s" % (self.user.full_name, self.channel.__unicode__()) + return "%s subscription of %s" % (self.name, self.user) @classmethod def _connect_mq(cls): @@ -195,13 +205,14 @@ def get_actions(self): def is_online(self): # TODO: Cache this method if self.channel.typ == 10: - return self.channel.subscriber_set.objects.exclude(user=self.user).get().user.is_online() - + return self.channel.subscriber_set.objects.exclude( + user=self.user).get().user.is_online() def unread_count(self): # FIXME: track and return actual unread message count if self.last_seen_msg_time: - return self.channel.message_set.objects.filter(timestamp__lt=self.last_seen_msg_time).count() + return self.channel.message_set.objects.filter( + timestamp__lt=self.last_seen_msg_time).count() else: self.channel.message_set.objects.filter().count() @@ -215,7 +226,7 @@ def create_exchange(self): to be safe we always call it before binding to the channel we currently subscribe """ channel = self._connect_mq() - channel.exchange_declare(exchange='prv_%s' % self.user.key.lower(), + channel.exchange_declare(exchange=self.user.prv_exchange, exchange_type='fanout', durable=True) @@ -236,11 +247,11 @@ def post_creation(self): self.create_exchange() self.bind_to_channel() + def pre_creation(self): if not self.name: self.name = self.channel.name - MSG_TYPES = ( (1, "Info Notification"), (11, "Error Notification"), @@ -287,7 +298,6 @@ def get_actions_for(self, user): ('Edit', '_zops_edit_message') ]) - def serialize(self, user=None): """ Serializes message for given user. @@ -306,7 +316,7 @@ def serialize(self, user=None): 'type': self.typ, 'updated_at': self.updated_at, 'timestamp': self.timestamp.strftime(DATE_TIME_FORMAT), - 'is_update': self.exist, + 'is_update': hasattr(self, 'unsaved'), 'attachments': [attachment.serialize() for attachment in self.attachment_set], 'title': self.msg_title, 'sender_name': self.sender.full_name, @@ -330,7 +340,7 @@ def _republish(self): body=json.dumps(self.serialize())) def pre_save(self): - if self.exist: + if not hasattr(self, 'unsaved'): self._republish() diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 26f14d44..fca75f07 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -295,6 +295,10 @@ def create_channel(current): description=current.input['description'], owner=current.user, typ=15).save() + sbs, new = Subscriber.objects.get_or_create(user=channel.owner, + channel=channel, + can_manage=True, + can_leave=False) current.output = { 'channel_key': channel.key, 'status': 'OK', @@ -600,7 +604,7 @@ def pin_channel(current): } """ try: - Subscriber(current).objects.get(user_id=current.user_id, + Subscriber(current).objects.filter(user_id=current.user_id, channel_id=current.input['channel_key']).update(pinned=True) current.output = {'status': 'OK', 'code': 200} except ObjectDoesNotExist: diff --git a/zengine/tornado_server/ws_to_queue.py b/zengine/tornado_server/ws_to_queue.py index 037333e3..ec7d9aa3 100644 --- a/zengine/tornado_server/ws_to_queue.py +++ b/zengine/tornado_server/ws_to_queue.py @@ -235,7 +235,7 @@ def unregister_websocket(self, sess_id): def create_out_channel(self, sess_id, user_id): def _on_output_channel_creation(channel): def _on_output_queue_decleration(queue): - channel.basic_consume(self.on_message, queue=sess_id) + channel.basic_consume(self.on_message, queue=sess_id, consumer_tag=user_id) log.debug("BIND QUEUE TO WS Q.%s on Ch.%s WS.%s" % (sess_id, channel.consumer_tags[0], user_id)) @@ -263,7 +263,7 @@ def publish_incoming_message(self, message, sess_id): body=json_encode(message)) def on_message(self, channel, method, header, body): - user_id = method.exchange[4:] + user_id = method.consumer_tag log.debug("WS RPLY for %s: %s" % (user_id, body)) if user_id in self.websockets: log.info("write msg to client") diff --git a/zengine/wf_daemon.py b/zengine/wf_daemon.py index 4649ac15..5ad8efd7 100755 --- a/zengine/wf_daemon.py +++ b/zengine/wf_daemon.py @@ -175,7 +175,7 @@ def handle_message(self, ch, method, properties, body): def send_output(self, output): # TODO: This is ugly, we should separate login process - log.debug("SEND_OUTPUT: %s" % output) + # log.debug("SEND_OUTPUT: %s" % output) if self.current.user_id is None or 'login_process' in output: self.client_queue.send_to_default_exchange(self.sessid, output) else: From bc58f0521a0d95a0445f2502b65c0a8abc86fcf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Thu, 21 Jul 2016 19:57:31 +0300 Subject: [PATCH 43/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/model.py | 34 +++++++++++++--------- zengine/messaging/views.py | 59 ++++++++++++++++++++++++-------------- 2 files changed, 58 insertions(+), 35 deletions(-) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index c3ef6844..750a2fde 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -13,6 +13,7 @@ from pyoko import Model, field, ListNode from pyoko.conf import settings +from pyoko.db.adapter.db_riak import BlockSave from pyoko.exceptions import IntegrityError from pyoko.fields import DATE_TIME_FORMAT from pyoko.lib.utils import get_object_from_path @@ -91,15 +92,22 @@ def get_or_create_direct_channel(cls, initiator_key, receiver_key): channel = existing[0] else: channel_name = '%s_%s' % (initiator_key, receiver_key) - channel = cls(is_direct=True, code_name=channel_name, typ=10).save() - Subscriber.objects.get_or_create(channel=channel, - user_id=initiator_key, - name=receiver_name) - Subscriber.objects.get_or_create(channel=channel, - user_id=receiver_key, - name=UserModel.objects.get(initiator_key).full_name) + channel = cls(is_direct=True, code_name=channel_name, typ=10).blocking_save() + with BlockSave(Subscriber): + Subscriber.objects.get_or_create(channel=channel, + user_id=initiator_key, + name=receiver_name) + Subscriber.objects.get_or_create(channel=channel, + user_id=receiver_key, + name=UserModel.objects.get(initiator_key).full_name) return channel, receiver_name + def get_avatar(self, user): + if self.typ == 10: + return self.subscriber_set.objects.exclude(user=user)[0].user.get_avatar_url() + else: + return None + @classmethod def add_message(cls, channel_key, body, title=None, sender=None, url=None, typ=2, receiver=None): @@ -112,6 +120,9 @@ def add_message(cls, channel_key, body, title=None, sender=None, url=None, typ=2 body=json.dumps(msg_object.serialize())) return msg_object.save() + def get_subscription_for_user(self, user_id): + return self.subscriber_set.objects.get(user_id=user_id) + def get_last_messages(self): # TODO: Try to refactor this with https://github.com/rabbitmq/rabbitmq-recent-history-exchange return self.message_set.objects.filter().set_params(sort="timestamp asc")[:20] @@ -132,11 +143,6 @@ def create_exchange(self): exchange_type='fanout', durable=True) - def subscribe_owner(self): - sbs, new = Subscriber.objects.get_or_create(user=self.owner, - channel=self, - can_manage=True, - can_leave=False) def pre_creation(self): if not self.code_name: @@ -183,6 +189,7 @@ class Subscriber(Model): def __unicode__(self): return "%s subscription of %s" % (self.name, self.user) + @classmethod def _connect_mq(cls): if cls.mq_connection is None or cls.mq_connection.is_closed: @@ -191,7 +198,8 @@ def _connect_mq(cls): def get_actions(self): actions = [ - ('Pin', '_zops_pin_channel') + ('Pin', '_zops_pin_channel'), + # ('Mute', '_zops_mute_channel'), ] if self.channel.owner == self.user or self.can_manage: actions.extend([ diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index fca75f07..b9348f03 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -7,6 +7,7 @@ # This file is licensed under the GNU General Public License v3 # (GPLv3). See LICENSE.txt for details. from pyoko.conf import settings +from pyoko.db.adapter.db_riak import BlockSave from pyoko.exceptions import ObjectDoesNotExist from pyoko.lib.utils import get_object_from_path from zengine.lib.exceptions import HTTPError @@ -111,8 +112,9 @@ def create_message(current): """ msg = current.input['message'] - msg_obj = Channel.add_message(msg['channel'], body=msg['body'], typ=msg['type'], sender=current.user, - title=msg['title'], receiver=msg['receiver'] or None) + msg_obj = Channel.add_message(msg['channel'], body=msg['body'], typ=msg['type'], + sender=current.user, + title=msg['title'], receiver=msg['receiver'] or None) current.output = { 'msg_key': msg_obj.key, 'status': 'OK', @@ -125,7 +127,7 @@ def create_message(current): file=atch['content'], description=atch['description'], typ=typ).save() -def show_channel(current): +def show_channel(current, waited=False): """ Initial display of channel content. Returns channel description, members, no of members, last 20 messages etc. @@ -155,8 +157,12 @@ def show_channel(current): } """ ch = Channel(current).objects.get(current.input['channel_key']) + sbs = ch.get_subscription_for_user(current.user_id) current.output = {'channel_key': current.input['channel_key'], 'description': ch.description, + 'name': sbs.name, + 'actions': sbs.get_actions(), + 'avatar_url': ch.get_avatar(current.user), 'no_of_members': len(ch.subscriber_set), 'member_list': [{'name': sb.user.full_name, 'is_online': sb.user.is_online(), @@ -295,15 +301,17 @@ def create_channel(current): description=current.input['description'], owner=current.user, typ=15).save() - sbs, new = Subscriber.objects.get_or_create(user=channel.owner, - channel=channel, - can_manage=True, - can_leave=False) - current.output = { - 'channel_key': channel.key, - 'status': 'OK', + with BlockSave(Subscriber): + Subscriber.objects.get_or_create(user=channel.owner, + channel=channel, + can_manage=True, + can_leave=False) + current.input['channel_key'] = channel.key + show_channel(current) + current.output.update({ + 'status': 'Created', 'code': 201 - } + }) def add_members(current): @@ -417,9 +425,14 @@ def search_user(current): 'status': 'OK', 'code': 201 } - for user in UserModel(current).objects.search_on(*settings.MESSAGING_USER_SEARCH_FIELDS, - contains=current.input['query']): - current.output['results'].append((user.full_name, user.key, user.get_avatar_url())) + qs = UserModel(current).objects.exclude(key=current.user_id).search_on( + *settings.MESSAGING_USER_SEARCH_FIELDS, + contains=current.input['query']) + # FIXME: somehow exclude(key=current.user_id) not working with search_on() + + for user in qs: + if user.key != current.user_id: + current.output['results'].append((user.full_name, user.key, user.get_avatar_url())) def search_unit(current): @@ -446,7 +459,7 @@ def search_unit(current): 'status': 'OK', 'code': 201 } - for user in UnitModel(current).objects.search_on(settings.MESSAGING_UNIT_SEARCH_FIELDS, + for user in UnitModel(current).objects.search_on(*settings.MESSAGING_UNIT_SEARCH_FIELDS, contains=current.input['query']): current.output['results'].append((user.name, user.key)) @@ -472,13 +485,14 @@ def create_direct_channel(current): 'name': string, # name of subscribed channel } """ - channel, sub_name = Channel.get_or_create_direct_channel(current.user_id, current.input['user_key']) - current.output = { - 'channel_key': channel.key, - 'name': sub_name, - 'status': 'OK', + channel, sub_name = Channel.get_or_create_direct_channel(current.user_id, + current.input['user_key']) + current.input['channel_key'] = channel.key + show_channel(current) + current.output.update({ + 'status': 'Created', 'code': 201 - } + }) def find_message(current): @@ -605,7 +619,8 @@ def pin_channel(current): """ try: Subscriber(current).objects.filter(user_id=current.user_id, - channel_id=current.input['channel_key']).update(pinned=True) + channel_id=current.input['channel_key']).update( + pinned=True) current.output = {'status': 'OK', 'code': 200} except ObjectDoesNotExist: raise HTTPError(404, "") From 62faa27fb136f53c0acea6278a1b648effb0236b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Thu, 21 Jul 2016 20:03:10 +0300 Subject: [PATCH 44/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/views.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index b9348f03..cc3947ce 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -151,6 +151,7 @@ def show_channel(current, waited=False): 'is_online': bool, 'avatar_url': string, }], + 'name': string, 'last_messages': [MSG_DICT] 'status': 'OK', 'code': 200 @@ -292,6 +293,15 @@ def create_channel(current): # response: { + 'description': string, + 'name': string, + 'no_of_members': int, + 'member_list': [ + {'name': string, + 'is_online': bool, + 'avatar_url': string, + }], + 'last_messages': [MSG_DICT] 'status': 'Created', 'code': 201, 'channel_key': key, # of just created channel @@ -479,6 +489,14 @@ def create_direct_channel(current): # response: { + 'description': string, + 'no_of_members': int, + 'member_list': [ + {'name': string, + 'is_online': bool, + 'avatar_url': string, + }], + 'last_messages': [MSG_DICT] 'status': 'Created', 'code': 201, 'channel_key': key, # of just created channel From f384d1ccc37271df37380cf6b95c35266eb9da8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Thu, 21 Jul 2016 20:51:42 +0300 Subject: [PATCH 45/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- tests/async_amqp/messaging_tests.py | 28 +++++++++---------- zengine/lib/concurrent_amqp_test_client.py | 31 +++++++++++----------- zengine/messaging/views.py | 26 ++++++++++-------- 3 files changed, 44 insertions(+), 41 deletions(-) diff --git a/tests/async_amqp/messaging_tests.py b/tests/async_amqp/messaging_tests.py index 254b4ad4..21a65a42 100644 --- a/tests/async_amqp/messaging_tests.py +++ b/tests/async_amqp/messaging_tests.py @@ -16,20 +16,20 @@ def test_channel_list(self): def test_search_user(self): self.post('ulakbus', {"view": "_zops_search_user", "query": "x"}) - def test_show_channel(self): - self.post('ulakbus', - {"view": "_zops_show_channel", - 'channel_key': 'iG4mvjQrfkvTDvM6Jk56X5ILoJ_CoqwpemOHnknn3hYu1BlAghb3dm'}) - - def test_create_message(self): - self.post('ulakbus', - {"view": "_zops_create_message", - "message": dict( - body='test_body', title='testtitle', - channel='iG4mvjQrfkvTDvM6Jk56X5ILoJ_CoqwpemOHnknn3hYu1BlAghb3dm', - receiver='', - type=2 - )}) + # def test_show_channel(self): + # self.post('ulakbus', + # {"view": "_zops_show_channel", + # 'channel_key': 'iG4mvjQrfkvTDvM6Jk56X5ILoJ_CoqwpemOHnknn3hYu1BlAghb3dm'}) + # + # def test_create_message(self): + # self.post('ulakbus', + # {"view": "_zops_create_message", + # "message": dict( + # body='test_body', title='testtitle', + # channel='iG4mvjQrfkvTDvM6Jk56X5ILoJ_CoqwpemOHnknn3hYu1BlAghb3dm', + # receiver='', + # type=2 + # )}) def main(): diff --git a/zengine/lib/concurrent_amqp_test_client.py b/zengine/lib/concurrent_amqp_test_client.py index ffb7fdda..3f8d4bd0 100644 --- a/zengine/lib/concurrent_amqp_test_client.py +++ b/zengine/lib/concurrent_amqp_test_client.py @@ -17,7 +17,7 @@ import inspect import uuid -from pprint import pprint +from pprint import pprint, pformat import pika from tornado.escape import json_encode, json_decode @@ -93,13 +93,9 @@ def backend_to_client(self, body): from backend to client """ body = json_decode(body) - try: - self.message_callbacks[body['callbackID']](body) - except KeyError: - print("No cb for %s" % body['callbackID']) - print("CB HELL %s" % self.message_callbacks) - self.message_stack[body['callbackID']] = body - log.info("WRITE MESSAGE TO CLIENT:\n%s" % (body,)) + self.message_stack[body['callbackID']] = body + self.message_callbacks[body['callbackID']](body) + log.info("WRITE MESSAGE TO CLIENT:\n%s" % (pformat(body),)) def client_to_backend(self, message, callback, caller_fn_name): """ @@ -112,7 +108,6 @@ def cb(res): print("API Request: %s :: %s\n" % (caller_fn_name, 'PASS' if result else 'FAIL!')) # self.message_callbacks[cbid] = lambda res: callable(res, message) self.message_callbacks[cbid] = cb - print(caller_fn_name, self.message_callbacks) log.info("GOT MESSAGE FOR BACKEND %s: %s" % (self.sess_id, message)) self.queue_manager.redirect_incoming_message(self.sess_id, message, self.request) @@ -169,13 +164,17 @@ def stc(self, response, request=None): response: request: """ - if not response['code'] in (200, 201): - print("FAILED: Response not successful: \n") - if not self.process_error_reponse(response): - print("\nRESP:\n%s") - print("\nREQ:\n %s" % (response, request)) - else: - return True + try: + if not response['code'] in (200, 201): + print("FAILED: Response not successful: \n") + if not self.process_error_reponse(response): + print("\nRESP:\n%s") + print("\nREQ:\n %s" % (response, request)) + else: + return True + except Exception as e: + log.exception("\n===========>\nFAILED API REQUEST\n<===========\n%s\n" % e) + log.info("Response: \n%s\n\n" % response) def pstc(self, response, request=None): """ diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index cc3947ce..ce560425 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -117,7 +117,7 @@ def create_message(current): title=msg['title'], receiver=msg['receiver'] or None) current.output = { 'msg_key': msg_obj.key, - 'status': 'OK', + 'status': 'Created', 'code': 201 } if 'attachment' in msg: @@ -170,7 +170,9 @@ def show_channel(current, waited=False): 'avatar_url': sb.user.get_avatar_url() } for sb in ch.subscriber_set.objects.filter()], 'last_messages': [msg.serialize(current.user) - for msg in ch.get_last_messages()] + for msg in ch.get_last_messages()], + 'status': 'OK', + 'code': 200 } @@ -265,15 +267,17 @@ def list_channels(current): },] } """ - current.output['channels'] = [ - {'name': sbs.name, - 'key': sbs.channel.key, - 'type': sbs.channel.typ, - 'read_only': sbs.read_only, - 'is_online': sbs.is_online(), - 'actions': sbs.get_actions(), - 'unread': sbs.unread_count()} for sbs in - current.user.subscriptions.objects.filter(is_visible=True)] + current.output = { + 'status': 'OK', + 'code': 200, + 'channels': [{'name': sbs.name, + 'key': sbs.channel.key, + 'type': sbs.channel.typ, + 'read_only': sbs.read_only, + 'is_online': sbs.is_online(), + 'actions': sbs.get_actions(), + 'unread': sbs.unread_count()} for sbs in + current.user.subscriptions.objects.filter(is_visible=True)]} def create_channel(current): From 899f02cf73a221765372815cb5f4c5602268b318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Fri, 22 Jul 2016 01:35:51 +0300 Subject: [PATCH 46/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- tests/async_amqp/messaging_tests.py | 41 ++++++++++-------- zengine/lib/concurrent_amqp_test_client.py | 36 +++++++++++++--- zengine/messaging/model.py | 13 ++++-- zengine/messaging/views.py | 48 +++++++++++----------- 4 files changed, 89 insertions(+), 49 deletions(-) diff --git a/tests/async_amqp/messaging_tests.py b/tests/async_amqp/messaging_tests.py index 21a65a42..02ce76bf 100644 --- a/tests/async_amqp/messaging_tests.py +++ b/tests/async_amqp/messaging_tests.py @@ -11,25 +11,32 @@ class TestCase(ConcurrentTestCase): def test_channel_list(self): - self.post('ulakbus', {"view": "_zops_list_channels"}) + self.post('ulakbus', dict(view="_zops_list_channels"), self.show_channel) def test_search_user(self): - self.post('ulakbus', {"view": "_zops_search_user", "query": "x"}) - - # def test_show_channel(self): - # self.post('ulakbus', - # {"view": "_zops_show_channel", - # 'channel_key': 'iG4mvjQrfkvTDvM6Jk56X5ILoJ_CoqwpemOHnknn3hYu1BlAghb3dm'}) - # - # def test_create_message(self): - # self.post('ulakbus', - # {"view": "_zops_create_message", - # "message": dict( - # body='test_body', title='testtitle', - # channel='iG4mvjQrfkvTDvM6Jk56X5ILoJ_CoqwpemOHnknn3hYu1BlAghb3dm', - # receiver='', - # type=2 - # )}) + self.post('ulakbus', + dict(view="_zops_search_user", query="x")) + + def show_channel(self, res, req): + ch_key = res['channels'][0]['key'] + self.post('ulakbus', + dict(view="_zops_show_channel", channel_key=ch_key), + self.create_message) + + + def create_message(self, res, req): + self.post('ulakbus', + {"view": "_zops_create_message", + "message": dict( + body='test_body', title='testtitle', + channel=res['channel_key'], + receiver='', + type=2 + )}) + + def cmd_message(self, res, req=None): + print("MESSAGE RECEIVED") + print(res) def main(): diff --git a/zengine/lib/concurrent_amqp_test_client.py b/zengine/lib/concurrent_amqp_test_client.py index 3f8d4bd0..68f21f82 100644 --- a/zengine/lib/concurrent_amqp_test_client.py +++ b/zengine/lib/concurrent_amqp_test_client.py @@ -92,9 +92,18 @@ def backend_to_client(self, body): """ from backend to client """ - body = json_decode(body) - self.message_stack[body['callbackID']] = body - self.message_callbacks[body['callbackID']](body) + try: + body = json_decode(body) + if 'callbackID' in body: + self.message_stack[body['callbackID']] = body + self.message_callbacks[body['callbackID']](body) + elif 'cmd' in body: + self.message_callbacks[body['cmd']](body) + except: + import traceback + print("\n") + traceback.print_exc() + log.info("WRITE MESSAGE TO CLIENT:\n%s" % (pformat(body),)) def client_to_backend(self, message, callback, caller_fn_name): @@ -104,8 +113,13 @@ def client_to_backend(self, message, callback, caller_fn_name): cbid = uuid.uuid4().hex message = json_encode({"callbackID": cbid, "data": message}) def cb(res): + print("API Request: %s :: " % caller_fn_name, end='') result = callback(res, message) - print("API Request: %s :: %s\n" % (caller_fn_name, 'PASS' if result else 'FAIL!')) + if ConcurrentTestCase.stc == callback and not result: + FAIL = 'FAIL' + else: + FAIL = '--> %s' % callback.__name__ + print('PASS' if result else FAIL) # self.message_callbacks[cbid] = lambda res: callable(res, message) self.message_callbacks[cbid] = cb log.info("GOT MESSAGE FOR BACKEND %s: %s" % (self.sess_id, message)) @@ -126,6 +140,8 @@ def __init__(self, queue_manager): self.clients = {} self.make_client('ulakbus') self.run_tests() + self.cmds = {} + self.register_cmds() def make_client(self, username): """ @@ -142,14 +158,24 @@ def make_client(self, username): def post(self, username, data, callback=None): if username not in self.clients: self.make_client(username) + self.clients[username].message_callbacks.update(self.cmds) callback = callback or self.stc view_name = data['view'] if 'view' in data else sys._getframe(1).f_code.co_name self.clients[username].client_to_backend(data, callback, view_name) + def register_cmds(self): + for name in sorted(self.__class__.__dict__): + if name.startswith("cmd_"): + self.cmds[name[4:]] = getattr(self, name) + def run_tests(self): for name in sorted(self.__class__.__dict__): if name.startswith("test_"): - getattr(self, name)() + try: + getattr(self, name)() + except: + import traceback + traceback.print_exc() def process_error_reponse(self, resp): if 'error' in resp: diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 750a2fde..c2f276af 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -125,7 +125,7 @@ def get_subscription_for_user(self, user_id): def get_last_messages(self): # TODO: Try to refactor this with https://github.com/rabbitmq/rabbitmq-recent-history-exchange - return self.message_set.objects.filter().set_params(sort="timestamp asc")[:20] + return self.message_set.objects.filter().set_params(sort="updated_at desc")[:20] @classmethod def _connect_mq(cls): @@ -297,7 +297,13 @@ class Message(Model): url = field.String("URL") def get_actions_for(self, user): - actions = [('Favorite', '_zops_favorite_message')] + actions = [] + if Favorite.objects.filter(user=user, + channel=self.channel, + message=self).count(): + actions.append(('Remove from favorites', '_zops_remove_from_favorites')) + else: + actions.append(('Add to favorites', '_zops_favorite_message')) if user: actions.extend([('Flag', '_zops_flag_message')]) if self.sender == user: @@ -305,6 +311,7 @@ def get_actions_for(self, user): ('Delete', '_zops_delete_message'), ('Edit', '_zops_edit_message') ]) + return actions def serialize(self, user=None): """ @@ -329,10 +336,10 @@ def serialize(self, user=None): 'title': self.msg_title, 'sender_name': self.sender.full_name, 'sender_key': self.sender.key, + 'channel_key': self.channel.key, 'cmd': 'message', 'avatar_url': self.sender.avatar, 'key': self.key, - 'actions': self.get_actions_for(user), } def __unicode__(self): diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index ce560425..1faa4b9e 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -10,6 +10,7 @@ from pyoko.db.adapter.db_riak import BlockSave from pyoko.exceptions import ObjectDoesNotExist from pyoko.lib.utils import get_object_from_path +from zengine.log import log from zengine.lib.exceptions import HTTPError from zengine.messaging.model import Channel, Attachment, Subscriber, Message, Favorite, \ FlaggedMessage @@ -34,15 +35,6 @@ 'avatar_url': string, 'key': key, 'cmd': 'message', - 'actions':[('action name', 'view name'), - ('Add to Favorite', '_zops_add_to_favorites'), # applicable to everyone - - # Additional actions should be retrieved - # from "_zops_get_message_actions" view. - ('Edit', '_zops_edit_message'), - ('Delete', '_zops_delete_message'), - - ] 'attachments': [{ 'description': string, 'file_name': string, @@ -169,11 +161,12 @@ def show_channel(current, waited=False): 'is_online': sb.user.is_online(), 'avatar_url': sb.user.get_avatar_url() } for sb in ch.subscriber_set.objects.filter()], - 'last_messages': [msg.serialize(current.user) - for msg in ch.get_last_messages()], + 'last_messages': [], 'status': 'OK', 'code': 200 } + for msg in ch.get_last_messages(): + current.output['last_messages'].insert(0, msg.serialize(current.user)) def channel_history(current): @@ -583,12 +576,18 @@ def delete_channel(current): 'code': 200 } """ + ch = Channel(current).objects.get(owner_id=current.user_id, + key=current.input['channel_key']) + for sbs in ch.subscriber_set.objects.filter(): + sbs.delete() + for msg in ch.message_set.objects.filter(): + msg.delete() try: - Channel(current).objects.get(owner_id=current.user_id, - key=current.input['channel_key']).delete() - current.output = {'status': 'Deleted', 'code': 200} - except ObjectDoesNotExist: - raise HTTPError(404, "") + ch.delete() + except: + log.exception("fix this!!!!!") + current.output = {'status': 'Deleted', 'code': 200} + def edit_channel(current): @@ -611,14 +610,15 @@ def edit_channel(current): 'code': 200 } """ - try: - Channel(current).objects.filter(owner_id=current.user_id, - key=current.input['channel_key'] - ).update(name=current.input['name'], - description=current.input['description']) - current.output = {'status': 'OK', 'code': 200} - except ObjectDoesNotExist: - raise HTTPError(404, "") + ch = Channel(current).objects.get(owner_id=current.user_id, + key=current.input['channel_key']) + ch.name = current.input['name'] + ch.description = current.input['description'] + ch.save() + for sbs in ch.subscriber_set.objects.filter(): + sbs.name = ch.name + sbs.save() + current.output = {'status': 'OK', 'code': 200} def pin_channel(current): From d9a27cdd24592b68e4e3cc708fb21aa391d5ccad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Fri, 22 Jul 2016 10:59:19 +0300 Subject: [PATCH 47/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/client_queue.py | 8 +++ zengine/messaging/lib.py | 21 ++++++-- zengine/messaging/model.py | 15 +++--- zengine/messaging/views.py | 105 +++++++++++++++++++++++-------------- zengine/settings.py | 1 + zengine/views/auth.py | 4 +- 6 files changed, 98 insertions(+), 56 deletions(-) diff --git a/zengine/client_queue.py b/zengine/client_queue.py index 5e2a5bc1..9c9dceb9 100644 --- a/zengine/client_queue.py +++ b/zengine/client_queue.py @@ -24,6 +24,14 @@ ) from zengine.log import log +def get_mq_connection(): + connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) + channel = connection.channel() + if not channel.is_open: + channel.open() + return connection, channel + + class ClientQueue(object): """ User AMQP queue manager diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py index 7a58a753..69a02bbd 100644 --- a/zengine/messaging/lib.py +++ b/zengine/messaging/lib.py @@ -12,7 +12,7 @@ from passlib.handlers.pbkdf2 import pbkdf2_sha512 from pyoko.conf import settings -from zengine.client_queue import BLOCKING_MQ_PARAMS +from zengine.client_queue import BLOCKING_MQ_PARAMS, get_mq_connection from zengine.lib.cache import Cache from zengine.log import log @@ -37,8 +37,7 @@ class BaseUser(object): @classmethod def _connect_mq(cls): if cls.mq_connection is None or cls.mq_connection.is_closed: - cls.mq_connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) - cls.mq_channel = cls.mq_connection.channel() + cls.mq_connection, cls.mq_channel = get_mq_connection() return cls.mq_channel def get_avatar_url(self): @@ -66,7 +65,19 @@ def set_password(self, raw_password): def is_online(self, status=None): if status is None: return ConnectionStatus(self.key).get() or False - ConnectionStatus(self.key).set(status) + else: + mq_channel = self._connect_mq() + for sbs in self.subscriptions.objects.filter(): + mq_channel.basic_publish(exchange=sbs.channel.key, + routing_key='', + body=json.dumps({ + 'cmd': 'user_status', + 'channel_key': sbs.channel.key, + 'channel_name': sbs.name, + 'avatar_url': self.get_avatar_url(), + 'is_online': status, + })) + ConnectionStatus(self.key).set(status) def encrypt_password(self): @@ -118,7 +129,7 @@ def prv_exchange(self): return 'prv_%s' % str(self.key).lower() def bind_private_channel(self, sess_id): - mq_channel = pika.BlockingConnection(BLOCKING_MQ_PARAMS).channel() + mq_channel = self._connect_mq() mq_channel.queue_declare(queue=sess_id, arguments={'x-expires': 40000}) log.debug("Binding private exchange to client queue: Q:%s --> E:%s" % (sess_id, self.prv_exchange)) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index c2f276af..9b1328e6 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -17,18 +17,12 @@ from pyoko.exceptions import IntegrityError from pyoko.fields import DATE_TIME_FORMAT from pyoko.lib.utils import get_object_from_path -from zengine.client_queue import BLOCKING_MQ_PARAMS +from zengine.client_queue import BLOCKING_MQ_PARAMS, get_mq_connection from zengine.lib.utils import to_safe_str UserModel = get_object_from_path(settings.USER_MODEL) -def get_mq_connection(): - connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS) - channel = connection.channel() - if not channel.is_open: - channel.open() - return connection, channel CHANNEL_TYPES = ( @@ -304,8 +298,13 @@ def get_actions_for(self, user): actions.append(('Remove from favorites', '_zops_remove_from_favorites')) else: actions.append(('Add to favorites', '_zops_favorite_message')) + + if user: - actions.extend([('Flag', '_zops_flag_message')]) + if FlaggedMessage.objects.filter(user=user, message=self).count(): + actions.append(('Remove Flag', '_zops_unflag_message')) + else: + actions.append(('Flag Message', '_zops_flag_message')) if self.sender == user: actions.extend([ ('Delete', '_zops_delete_message'), diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 1faa4b9e..f2a793ff 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -22,25 +22,35 @@ .. code-block:: python - MSG_DICT = {'content': string, - 'title': string, - 'timestamp': datetime, - 'updated_at': datetime, - 'is_update': boolean, # false for new messages - # true if this is an updated message - 'channel_key': key, - 'sender_name': string, - 'sender_key': key, - 'type': int, - 'avatar_url': string, - 'key': key, - 'cmd': 'message', - 'attachments': [{ - 'description': string, - 'file_name': string, - 'url': string, - },] - } + MSG_DICT = { + 'content': string, + 'title': string, + 'timestamp': datetime, + 'updated_at': datetime, + 'is_update': boolean, # false for new messages + # true if this is an updated message + 'channel_key': key, + 'sender_name': string, + 'sender_key': key, + 'type': int, + 'avatar_url': string, + 'key': key, + 'cmd': 'message', + 'attachments': [{ + 'description': string, + 'file_name': string, + 'url': string, + },] +} + + + USER_STATUS_UPDATE = { + 'cmd': 'user_status', + 'channel_key': key, + 'channel_name': string, + 'avatar_url': string, + 'is_online': boolean, + } """ @@ -589,7 +599,6 @@ def delete_channel(current): current.output = {'status': 'Deleted', 'code': 200} - def edit_channel(current): """ Update channel name or description @@ -714,28 +723,44 @@ def flag_message(current): # request: { 'view':'_zops_flag_message', - 'message': { - 'key': key - 'flag': boolean, # true for flagging - # false for unflagging - } + 'message_key': key, } # response: { ' - 'status': string, # 'OK' for success - 'code': int, # 200 for success + 'status': 'Created', + 'code': 201, + } + + """ + current.output = {'status': 'Created', 'code': 201} + FlaggedMessage.objects.get_or_create(user_id=current.user_id, + message_id=current.input['key']) + + +def unflag_message(current): + """ + remove flag of a message + + .. code-block:: python + + # request: + { + 'view':'_zops_flag_message', + 'key': key, + } + # response: + { + ' + 'status': 'OK', + 'code': 200, } """ current.output = {'status': 'OK', 'code': 200} - if current.input['flag']: - FlaggedMessage.objects.get_or_create(current, - user_id=current.user_id, - message_id=current.input['key']) - else: - FlaggedMessage(current).objects.filter(user_id=current.user_id, - message_id=current.input['key']).delete() + + FlaggedMessage(current).objects.filter(user_id=current.user_id, + message_id=current.input['key']).delete() def get_message_actions(current): @@ -772,7 +797,7 @@ def add_to_favorites(current): # request: { 'view':'_zops_add_to_favorites, - 'message_key': key, + 'key': key, } # response: @@ -785,7 +810,7 @@ def add_to_favorites(current): """ msg = Message.objects.get(current.input['message_key']) current.output = {'status': 'Created', 'code': 201} - fav, new = Favorite.objects.get_or_create(user_id=current.user_id, message=msg['key']) + fav, new = Favorite.objects.get_or_create(user_id=current.user_id, message=msg) current.output['favorite_key'] = fav.key @@ -798,20 +823,20 @@ def remove_from_favorites(current): # request: { 'view':'_zops_remove_from_favorites, - 'message_key': key, + 'key': key, } # response: { - 'status': 'Deleted', + 'status': 'OK', 'code': 200 } """ try: - current.output = {'status': 'Deleted', 'code': 200} + current.output = {'status': 'OK', 'code': 200} Favorite(current).objects.get(user_id=current.user_id, - key=current.input['message_key']).delete() + key=current.input['key']).delete() except ObjectDoesNotExist: raise HTTPError(404, "") diff --git a/zengine/settings.py b/zengine/settings.py index d6ab49cc..ef0c0811 100644 --- a/zengine/settings.py +++ b/zengine/settings.py @@ -142,6 +142,7 @@ '_zops_delete_channel': 'zengine.messaging.views.delete_channel', '_zops_pin_channel': 'zengine.messaging.views.pin_channel', '_zops_flag_message': 'zengine.messaging.views.flag_message', + '_zops_unflag_message': 'zengine.messaging.views.unflag_message', # '_zops_': 'zengine.messaging.views.', } diff --git a/zengine/views/auth.py b/zengine/views/auth.py index d8571fdf..8f2bb95e 100644 --- a/zengine/views/auth.py +++ b/zengine/views/auth.py @@ -30,9 +30,7 @@ def logout(current): Args: current: :attr:`~zengine.engine.WFCurrent` object. """ - user_id = current.session.get('user_id') - if user_id: - KeepAlive(user_id).delete() + current.user.is_online(False) current.session.delete() From 9b95fbf0dc42cfc98ad34c3122cb6336297f29f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Fri, 22 Jul 2016 12:19:52 +0300 Subject: [PATCH 48/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/lib.py | 8 ++++---- zengine/messaging/model.py | 2 +- zengine/messaging/views.py | 11 ++++++----- zengine/tornado_server/server.py | 2 +- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py index 69a02bbd..e7ea46b3 100644 --- a/zengine/messaging/lib.py +++ b/zengine/messaging/lib.py @@ -68,9 +68,10 @@ def is_online(self, status=None): else: mq_channel = self._connect_mq() for sbs in self.subscriptions.objects.filter(): - mq_channel.basic_publish(exchange=sbs.channel.key, - routing_key='', - body=json.dumps({ + if sbs.channel.typ == 10: + mq_channel.basic_publish(exchange=sbs.channel.code_name, + routing_key='', + body=json.dumps({ 'cmd': 'user_status', 'channel_key': sbs.channel.key, 'channel_name': sbs.name, @@ -79,7 +80,6 @@ def is_online(self, status=None): })) ConnectionStatus(self.key).set(status) - def encrypt_password(self): """ encrypt password if not already encrypted """ if self.password and not self.password.startswith('$pbkdf2'): diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 9b1328e6..ed90656a 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -329,7 +329,7 @@ def serialize(self, user=None): 'content': self.body, 'type': self.typ, 'updated_at': self.updated_at, - 'timestamp': self.timestamp.strftime(DATE_TIME_FORMAT), + 'timestamp': self.updated_at, 'is_update': hasattr(self, 'unsaved'), 'attachments': [attachment.serialize() for attachment in self.attachment_set], 'title': self.msg_title, diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index f2a793ff..29ef5781 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -224,7 +224,7 @@ def report_last_seen_message(current): { 'view':'_zops_last_seen_msg', 'channel_key': key, - 'msg_key': key, + 'key': key, 'timestamp': datetime, } @@ -671,14 +671,15 @@ def delete_message(current): # response: { + 'key': key, 'status': 'OK', 'code': 200 } """ try: Message(current).objects.get(sender_id=current.user_id, - key=current.input['message_key']).delete() - current.output = {'status': 'Deleted', 'code': 200} + key=current.input['key']).delete() + current.output = {'status': 'Deleted', 'code': 200, 'key': current.input['key']} except ObjectDoesNotExist: raise HTTPError(404, "") @@ -772,7 +773,7 @@ def get_message_actions(current): # request: { 'view':'_zops_get_message_actions', - 'message_key': key, + 'key': key, } # response: { @@ -785,7 +786,7 @@ def get_message_actions(current): current.output = {'status': 'OK', 'code': 200, 'actions': Message.objects.get( - current.input['message_key']).get_actions_for(current.user)} + current.input['key']).get_actions_for(current.user)} def add_to_favorites(current): diff --git a/zengine/tornado_server/server.py b/zengine/tornado_server/server.py index 7c13e8b7..9753f57b 100644 --- a/zengine/tornado_server/server.py +++ b/zengine/tornado_server/server.py @@ -137,7 +137,7 @@ def post(self, view_name): (r'/(\w+)', HttpHandler), ] -app = web.Application(URL_CONFS, debug=DEBUG) +app = web.Application(URL_CONFS, debug=DEBUG, autoreload=False) def runserver(host=None, port=None): From e0a723f22a4537db3c171f150131c1ce0abe0460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Fri, 22 Jul 2016 12:35:56 +0300 Subject: [PATCH 49/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/views.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 29ef5781..524240f1 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -140,7 +140,7 @@ def show_channel(current, waited=False): # request: { 'view':'_zops_show_channel', - 'channel_key': key, + 'key': key, } # response: @@ -159,9 +159,9 @@ def show_channel(current, waited=False): 'code': 200 } """ - ch = Channel(current).objects.get(current.input['channel_key']) + ch = Channel(current).objects.get(current.input['key']) sbs = ch.get_subscription_for_user(current.user_id) - current.output = {'channel_key': current.input['channel_key'], + current.output = {'channel_key': current.input['key'], 'description': ch.description, 'name': sbs.name, 'actions': sbs.get_actions(), @@ -323,7 +323,7 @@ def create_channel(current): channel=channel, can_manage=True, can_leave=False) - current.input['channel_key'] = channel.key + current.input['key'] = channel.key show_channel(current) current.output.update({ 'status': 'Created', @@ -512,7 +512,7 @@ def create_direct_channel(current): """ channel, sub_name = Channel.get_or_create_direct_channel(current.user_id, current.input['user_key']) - current.input['channel_key'] = channel.key + current.input['key'] = channel.key show_channel(current) current.output.update({ 'status': 'Created', From da6cf4c6d661db401446023769253000484c3a02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Fri, 22 Jul 2016 13:57:55 +0300 Subject: [PATCH 50/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/model.py | 2 +- zengine/messaging/views.py | 33 +++++++++++++++++++-------------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index ed90656a..ee362659 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -216,7 +216,7 @@ def unread_count(self): return self.channel.message_set.objects.filter( timestamp__lt=self.last_seen_msg_time).count() else: - self.channel.message_set.objects.filter().count() + return self.channel.message_set.objects.filter().count() def create_exchange(self): """ diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 524240f1..d1cebb9d 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -161,7 +161,7 @@ def show_channel(current, waited=False): """ ch = Channel(current).objects.get(current.input['key']) sbs = ch.get_subscription_for_user(current.user_id) - current.output = {'channel_key': current.input['key'], + current.output = {'key': current.input['key'], 'description': ch.description, 'name': sbs.name, 'actions': sbs.get_actions(), @@ -202,12 +202,13 @@ def channel_history(current): current.output = { 'status': 'OK', 'code': 201, - 'messages': [ - msg.serialize(current.user) - for msg in Message.objects.filter(channel_id=current.input['channel_key'], - timestamp__lt=current.input['timestamp'])[:20]] + 'messages': [] } + for msg in Message.objects.filter(channel_id=current.input['channel_key'], + updated_at__lt=current.input['timestamp'])[:20]: + current.output['messages'].insert(0, msg.serialize(current.user)) + def report_last_seen_message(current): """ @@ -273,14 +274,18 @@ def list_channels(current): current.output = { 'status': 'OK', 'code': 200, - 'channels': [{'name': sbs.name, - 'key': sbs.channel.key, - 'type': sbs.channel.typ, - 'read_only': sbs.read_only, - 'is_online': sbs.is_online(), - 'actions': sbs.get_actions(), - 'unread': sbs.unread_count()} for sbs in - current.user.subscriptions.objects.filter(is_visible=True)]} + 'channels': []} + for sbs in current.user.subscriptions.objects.filter(is_visible=True): + try: + current.output['channels'].append({'name': sbs.name, + 'key': sbs.channel.key, + 'type': sbs.channel.typ, + 'read_only': sbs.read_only, + 'is_online': sbs.is_online(), + 'actions': sbs.get_actions(), + 'unread': sbs.unread_count()}) + except ObjectDoesNotExist: + sbs.delete() def create_channel(current): @@ -311,7 +316,7 @@ def create_channel(current): 'last_messages': [MSG_DICT] 'status': 'Created', 'code': 201, - 'channel_key': key, # of just created channel + 'key': key, # of just created channel } """ channel = Channel(name=current.input['name'], From efb956d277e2b1369221893b513c9608510fba4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Fri, 22 Jul 2016 14:10:59 +0300 Subject: [PATCH 51/61] rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/model.py | 2 +- zengine/messaging/views.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index ee362659..8c89810e 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -176,7 +176,7 @@ class Subscriber(Model): is_visible = field.Boolean("Show under user's channel list", default=True) can_manage = field.Boolean("Can manage this channel", default=False) can_leave = field.Boolean("Membership is not obligatory", default=True) - last_seen_msg_time = field.DateTime("Last seen message's time") + last_seen_msg_time = field.TimeStamp("Last seen message's time") # status = field.Integer("Status", choices=SUBSCRIPTION_STATUS) diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index d1cebb9d..6834edb7 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -238,7 +238,9 @@ def report_last_seen_message(current): Subscriber(current).objects.filter(channel_id=current.input['channel_key'], user_id=current.user_id ).update(last_seen_msg_time=current.input['timestamp']) - + current.output = { + 'status': 'OK', + 'code': 200} def list_channels(current): """ From 084c8361155af5a9758183d9901748f86c731fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Mon, 25 Jul 2016 14:11:55 +0300 Subject: [PATCH 52/61] added unread_count API view fixed unread calculation fixed BaseUser.send_notification rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/messaging/lib.py | 2 +- zengine/messaging/model.py | 2 +- zengine/messaging/views.py | 44 +++++++++++++++++++++++++++++++++++--- zengine/settings.py | 1 + 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/zengine/messaging/lib.py b/zengine/messaging/lib.py index e7ea46b3..7214ae9a 100644 --- a/zengine/messaging/lib.py +++ b/zengine/messaging/lib.py @@ -147,7 +147,7 @@ def send_notification(self, title, message, typ=1, url=None): """ - self.channel_set.channel.__class__.add_message( + self.created_channels.channel.add_message( channel_key=self.prv_exchange, body=message, title=title, diff --git a/zengine/messaging/model.py b/zengine/messaging/model.py index 8c89810e..500421c0 100644 --- a/zengine/messaging/model.py +++ b/zengine/messaging/model.py @@ -214,7 +214,7 @@ def unread_count(self): # FIXME: track and return actual unread message count if self.last_seen_msg_time: return self.channel.message_set.objects.filter( - timestamp__lt=self.last_seen_msg_time).count() + timestamp__gt=self.last_seen_msg_time).count() else: return self.channel.message_set.objects.filter().count() diff --git a/zengine/messaging/views.py b/zengine/messaging/views.py index 6834edb7..aeeffbc5 100644 --- a/zengine/messaging/views.py +++ b/zengine/messaging/views.py @@ -235,9 +235,10 @@ def report_last_seen_message(current): 'code': 200, } """ - Subscriber(current).objects.filter(channel_id=current.input['channel_key'], - user_id=current.user_id - ).update(last_seen_msg_time=current.input['timestamp']) + sbs = Subscriber(current).objects.filter(channel_id=current.input['channel_key'], + user_id=current.user_id)[0] + sbs.last_seen_msg_time=current.input['timestamp'] + sbs.save() current.output = { 'status': 'OK', 'code': 200} @@ -289,6 +290,43 @@ def list_channels(current): except ObjectDoesNotExist: sbs.delete() +def unread_count(current): + """ + Number of unread messages for current user + + + .. code-block:: python + + # request: + { + 'view':'_zops_unread_count', + } + + # response: + { + 'status': 'OK', + 'code': 200, + 'notifications': int, + 'messages': int, + } + """ + unread_ntf = 0 + unread_msg = 0 + for sbs in current.user.subscriptions.objects.filter(is_visible=True): + try: + if sbs.channel.key == current.user.prv_exchange: + unread_ntf += sbs.unread_count() + else: + unread_msg += sbs.unread_count() + except ObjectDoesNotExist: + sbs.delete() + current.output = { + 'status': 'OK', + 'code': 200, + 'notifications': unread_ntf, + 'messages': unread_msg + } + def create_channel(current): """ diff --git a/zengine/settings.py b/zengine/settings.py index ef0c0811..5796f08e 100644 --- a/zengine/settings.py +++ b/zengine/settings.py @@ -143,6 +143,7 @@ '_zops_pin_channel': 'zengine.messaging.views.pin_channel', '_zops_flag_message': 'zengine.messaging.views.flag_message', '_zops_unflag_message': 'zengine.messaging.views.unflag_message', + '_zops_unread_count': 'zengine.messaging.views.unread_count', # '_zops_': 'zengine.messaging.views.', } From 114a8d6285610289532051509bf16ab759dec6b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Mon, 25 Jul 2016 14:12:19 +0300 Subject: [PATCH 53/61] more testing work... rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- tests/async_amqp/messaging_tests.py | 20 +++++++++++++------- zengine/lib/concurrent_amqp_test_client.py | 4 ++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/async_amqp/messaging_tests.py b/tests/async_amqp/messaging_tests.py index 02ce76bf..6057bc8d 100644 --- a/tests/async_amqp/messaging_tests.py +++ b/tests/async_amqp/messaging_tests.py @@ -10,6 +10,9 @@ class TestCase(ConcurrentTestCase): + def __init__(self, *args, **kwargs): + super(TestCase, self).__init__(*args, **kwargs) + def test_channel_list(self): self.post('ulakbus', dict(view="_zops_list_channels"), self.show_channel) @@ -23,16 +26,19 @@ def show_channel(self, res, req): dict(view="_zops_show_channel", channel_key=ch_key), self.create_message) - def create_message(self, res, req): self.post('ulakbus', {"view": "_zops_create_message", - "message": dict( - body='test_body', title='testtitle', - channel=res['channel_key'], - receiver='', - type=2 - )}) + "message": dict( + body='test_body', title='testtitle', + channel=res['key'], + receiver='', + type=2 + )}) + + def cmd_user_status(self, res, req): + print("CMD: user_status:") + print(res) def cmd_message(self, res, req=None): print("MESSAGE RECEIVED") diff --git a/zengine/lib/concurrent_amqp_test_client.py b/zengine/lib/concurrent_amqp_test_client.py index 68f21f82..e0248aa5 100644 --- a/zengine/lib/concurrent_amqp_test_client.py +++ b/zengine/lib/concurrent_amqp_test_client.py @@ -98,7 +98,7 @@ def backend_to_client(self, body): self.message_stack[body['callbackID']] = body self.message_callbacks[body['callbackID']](body) elif 'cmd' in body: - self.message_callbacks[body['cmd']](body) + self.cmds[body['cmd']](body) except: import traceback print("\n") @@ -136,11 +136,11 @@ class ConcurrentTestCase(object): def __init__(self, queue_manager): log.info("ConcurrentTestCase class init with %s" % queue_manager) + self.cmds = {} self.queue_manager = queue_manager self.clients = {} self.make_client('ulakbus') self.run_tests() - self.cmds = {} self.register_cmds() def make_client(self, username): From e210ec86e2f43d9729b44c2d118b307c98394df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Tue, 26 Jul 2016 10:45:34 +0300 Subject: [PATCH 54/61] amqp based concurrent test framework, nearly finished rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- tests/async_amqp/messaging_tests.py | 18 +++++++++++------- zengine/lib/concurrent_amqp_test_client.py | 6 +++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/async_amqp/messaging_tests.py b/tests/async_amqp/messaging_tests.py index 6057bc8d..ba21c4d5 100644 --- a/tests/async_amqp/messaging_tests.py +++ b/tests/async_amqp/messaging_tests.py @@ -6,6 +6,8 @@ # # This file is licensed under the GNU General Public License v3 # (GPLv3). See LICENSE.txt for details. +from pprint import pprint + from zengine.lib.concurrent_amqp_test_client import ConcurrentTestCase, TestQueueManager @@ -18,15 +20,17 @@ def test_channel_list(self): def test_search_user(self): self.post('ulakbus', - dict(view="_zops_search_user", query="x")) + dict(view="_zops_search_user", + query="u")) def show_channel(self, res, req): ch_key = res['channels'][0]['key'] self.post('ulakbus', - dict(view="_zops_show_channel", channel_key=ch_key), - self.create_message) + dict(view="_zops_show_channel", + key=ch_key), + callback=self.create_message) - def create_message(self, res, req): + def create_message(self, res, req=None): self.post('ulakbus', {"view": "_zops_create_message", "message": dict( @@ -36,13 +40,13 @@ def create_message(self, res, req): type=2 )}) - def cmd_user_status(self, res, req): + def cmd_user_status(self, res, req=None): print("CMD: user_status:") - print(res) + pprint(res) def cmd_message(self, res, req=None): print("MESSAGE RECEIVED") - print(res) + pprint(res) def main(): diff --git a/zengine/lib/concurrent_amqp_test_client.py b/zengine/lib/concurrent_amqp_test_client.py index e0248aa5..234c5f1c 100644 --- a/zengine/lib/concurrent_amqp_test_client.py +++ b/zengine/lib/concurrent_amqp_test_client.py @@ -98,10 +98,10 @@ def backend_to_client(self, body): self.message_stack[body['callbackID']] = body self.message_callbacks[body['callbackID']](body) elif 'cmd' in body: - self.cmds[body['cmd']](body) + self.message_callbacks[body['cmd']](body) except: import traceback - print("\n") + print("\nException BODY: %s \n" % pformat(body)) traceback.print_exc() log.info("WRITE MESSAGE TO CLIENT:\n%s" % (pformat(body),)) @@ -137,11 +137,11 @@ class ConcurrentTestCase(object): def __init__(self, queue_manager): log.info("ConcurrentTestCase class init with %s" % queue_manager) self.cmds = {} + self.register_cmds() self.queue_manager = queue_manager self.clients = {} self.make_client('ulakbus') self.run_tests() - self.register_cmds() def make_client(self, username): """ From 6e59ebdc43c8d5a9b4452f980782798d6d9f0ba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Tue, 26 Jul 2016 10:45:50 +0300 Subject: [PATCH 55/61] fixed prepare_post rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/lib/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zengine/lib/test_utils.py b/zengine/lib/test_utils.py index d9a6a858..f0e0704e 100644 --- a/zengine/lib/test_utils.py +++ b/zengine/lib/test_utils.py @@ -94,7 +94,7 @@ def set_path(self, path, token=''): self.path = path self.token = token - def _prepare_post(self, **data): + def _prepare_post(self, data): """ by default data dict encoded as json and content type set as application/json From b803a2fd9bec5c6868c258045a8b919648dcdb6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Evren=20Esat=20=C3=96zkan?= Date: Tue, 26 Jul 2016 11:14:28 +0300 Subject: [PATCH 56/61] fixed lane_change_invitation rref #5367 rref #5366 ref zetaops/zengine#66 ref zetaops/zengine#65 --- zengine/receivers.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/zengine/receivers.py b/zengine/receivers.py index 152e0394..db3262bb 100644 --- a/zengine/receivers.py +++ b/zengine/receivers.py @@ -11,14 +11,10 @@ 'set_password', ] - - - from pyoko.conf import settings from zengine.dispatch.dispatcher import receiver from zengine.signals import lane_user_change, crud_post_save - DEFAULT_LANE_CHANGE_INVITE_MSG = { 'title': settings.MESSAGES['lane_change_invite_title'], 'body': settings.MESSAGES['lane_change_invite_body'], @@ -48,11 +44,11 @@ def send_message_for_lane_change(sender, *args, **kwargs): for recipient in owners: if not isinstance(recipient, UserModel): recipient = recipient.get_user() - Notify(recipient.key).set_message(title=_(msg_context['title']), - msg=_(msg_context['body']), - typ=Notify.TaskInfo, - url=current.get_wf_link() - ) + recipient.send_notification(title=_(msg_context['title']), + message=_(msg_context['body']), + typ=Notify.TaskInfo, + url=current.get_wf_link() + ) # encrypting password on save From a94253b0d4b60084fe73e4ea386f181b8c69af19 Mon Sep 17 00:00:00 2001 From: Ali Riza Keles Date: Mon, 18 Jul 2016 17:25:56 +0300 Subject: [PATCH 57/61] bump version 0.7.1 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index eb49d7c7..39e898a4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7 +0.7.1 From 7dc37f9bec985891a246fc6e35b041100402c440 Mon Sep 17 00:00:00 2001 From: Ali Riza Keles Date: Mon, 18 Jul 2016 18:27:52 +0300 Subject: [PATCH 58/61] setup.py updated --- setup.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 8d53451f..31ea8e13 100644 --- a/setup.py +++ b/setup.py @@ -11,12 +11,12 @@ author_email='info@zetaops.io', description='BPMN workflow based web service framework with advanced ' 'permissions and extensible CRUD features', - install_requires=['beaker', 'passlib', 'falcon', 'beaker_extensions', - 'redis', 'SpiffWorkflow', 'pyoko', 'tornado', 'pika'], + install_requires=['beaker', 'passlib', 'falcon', 'beaker_extensions', 'lazy_object_proxy' + 'redis', 'enum34', 'werkzeug', 'celery', 'SpiffWorkflow', 'pyoko', + 'tornado', 'pika'], dependency_links=[ 'git+https://github.com/didip/beaker_extensions.git#egg=beaker_extensions', 'git+https://github.com/zetaops/SpiffWorkflow.git#egg=SpiffWorkflow', - #'git+https://github.com/zetaops/pyoko.git#egg=pyoko', ], package_data={ 'zengine': ['diagrams/*.bpmn'], @@ -25,3 +25,11 @@ 'json', 'bpmn', 'workflow', 'web service', 'orm', 'nosql', 'bpmn 2', 'crud', 'scaffolding'], ) + + + + + + + + From 57b86568e7b0acb089b513d22b6269ba3cc76f0d Mon Sep 17 00:00:00 2001 From: Ali Riza Keles Date: Mon, 18 Jul 2016 18:29:11 +0300 Subject: [PATCH 59/61] bump version 0.7.2 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 39e898a4..2c0a9c7b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.1 +v0.7.2 From f32e109866adc5d720c6f59f54063c5531d5cbc5 Mon Sep 17 00:00:00 2001 From: Ali Riza Keles Date: Tue, 19 Jul 2016 12:07:06 +0300 Subject: [PATCH 60/61] bump version 0.7.3 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2c0a9c7b..3d105a6f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.7.2 +v0.7.3 From 62d53f2abebca799bbb64c5a48ad9d45bcbd0957 Mon Sep 17 00:00:00 2001 From: dyrnade Date: Tue, 19 Jul 2016 14:51:04 +0300 Subject: [PATCH 61/61] added forgotten comma --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 31ea8e13..989b2e4e 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ author_email='info@zetaops.io', description='BPMN workflow based web service framework with advanced ' 'permissions and extensible CRUD features', - install_requires=['beaker', 'passlib', 'falcon', 'beaker_extensions', 'lazy_object_proxy' + install_requires=['beaker', 'passlib', 'falcon', 'beaker_extensions', 'lazy_object_proxy', 'redis', 'enum34', 'werkzeug', 'celery', 'SpiffWorkflow', 'pyoko', 'tornado', 'pika'], dependency_links=[