Release 2024.12.0

.circleci/config.yml

@@ -19,6 +19,7 @@ jobs:
             else
               python3 -m venv .venv
               source .venv/bin/activate
+              pip install --upgrade pip
               pip install -e .[dev-pinned,pinned]
             fi
       - save_cache:

.github/ISSUE_TEMPLATE/bug-report.md

@@ -34,3 +34,7 @@ labels: 'bug'
   ```
 
 </details>
+
+### Related Issues
+<!-- A collection of all issues that are related to this issue and could be useful. -->
+<!-- Please also include the related integreat-app issues, if possible. -->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -28,3 +28,8 @@ As a [type of user] I want [goals or objectives] so that [values or benefits]. -
 
 ### Design Requirements
 <!-- If the customization includes input from our design team, the detailed requirements will be collected here. Note: These will exist mainly in German to simplify internal communication. -->
+
+### Related Issues
+<!-- A collection of all issues that are related to this issue and could be useful. -->
+<!-- Please also include the related integreat-app issues, if possible. -->
+<!-- If the app issue does not exist yet, ping Toni on Mattermost to create one and then link it here. -->

.gitignore

@@ -68,3 +68,6 @@ integreat_cms/xliff/download
 
 # Postgres folder
 .postgres
+
+# Celery
+celerybeat-schedule.db

docs/src/debugging.rst

@@ -45,6 +45,40 @@ VSCodium
 ^^^^^^^^
 Coming soon(TM)?
 
+PyCharm (Professional)
+^^^^^^^^^^^^^^^^^^^^^^
+
+PyCharm Professional has built-in support for django projects, but it needs some configuration to work with the integreat CMS.
+This is unfortunately not the case for the free edition of PyCharm. Students can get a free license for the professional version, however.
+
+Enable Django Support
+---------------------
+
+#. Go to settings → Languages & Frameworks → Django
+#. Click the ``Enable Django Support`` Checkbox
+#. Set the root ``integreat_cms`` directory as the project root
+#. For ``Settings`` use ``integreat_cms/core/docker_settings.py``
+#. Check ``Do not use Django test runner``
+#. For ``Manage script``, use ``integreat_cms/integreat-cms-cli``. If PyCharm does not let you select this script, because it does not end in .py, you can manually specify it in ``.idea/integreat-cms.iml``.
+
+Your configuration should now look similar to this:
+
+   .. image:: images/debugging/debug-pycharm-01-django-config.png
+      :alt: Django configuration
+
+Create a Run Configuration
+--------------------------
+
+#. Create a new ``Django Server`` run configuration
+#. Use a different port (For example 8001) to avoid conflicts with the non-debug server at port 8000
+#. At `Environment Variables`, add these: ``PYTHONUNBUFFERED=1;DJANGO_SETTINGS_MODULE=integreat_cms.core.docker_settings;INTEGREAT_CMS_DEBUG=True``
+
+Start Debugging
+---------------
+
+#. First execute the ``./tools/run.sh`` to make sure that the database is available and all assets are compiled.
+#. Once the server has started, you can start debugging by launching the run configuration
+
 Neovim
 ^^^^^^
 

integreat_cms/api/v3/chat/user_chat.py

@@ -7,7 +7,6 @@
 import json
 import logging
 import random
-import socket
 from typing import TYPE_CHECKING
 
 from django.http import HttpResponse, JsonResponse
@@ -17,8 +16,8 @@
 
 from ....cms.models import ABTester, AttachmentMap, Language, Region, UserChat
 from ...decorators import json_response
-from .chat_bot import ChatBot
-from .zammad_api import ZammadChatAPI
+from .utils.chat_bot import process_answer, process_user_message
+from .utils.zammad_api import ZammadChatAPI
 
 if TYPE_CHECKING:
     from django.http import HttpRequest
@@ -227,15 +226,13 @@ def zammad_webhook(request: HttpRequest) -> JsonResponse:
     """
     Receive webhooks from Zammad to update the latest article translation
     """
-    zammad_url = (
-        f"https://{socket.getnameinfo((request.META.get('REMOTE_ADDR'), 0), 0)[0]}"
+    region = get_object_or_404(
+        Region, zammad_webhook_token=request.GET.get("token", None)
     )
-    region = get_object_or_404(Region, zammad_url=zammad_url)
-    client = ZammadChatAPI(region)
+    if not region.integreat_chat_enabled:
+        return JsonResponse({"status": "Integreat Chat disabled"})
     webhook_message = json.loads(request.body)
     message_text = webhook_message["article"]["body"]
-    zammad_chat = UserChat.objects.get(zammad_id=webhook_message["ticket"]["id"])
-    chat_bot = ChatBot()
 
     actions = []
     if webhook_message["article"]["internal"]:
@@ -249,34 +246,14 @@ def zammad_webhook(request: HttpRequest) -> JsonResponse:
         webhook_message["article"]["created_by"]["login"]
         == "[email protected]"
     ):
-        actions.append("question translation")
-        client.send_message(
-            zammad_chat.zammad_id,
-            chat_bot.automatic_translation(
-                message_text, zammad_chat.language.slug, region.default_language.slug
-            ),
-            True,
-            True,
+        actions.append("question translation queued")
+        process_user_message.apply_async(
+            args=[message_text, region.slug, webhook_message["ticket"]["id"]]
         )
-        if answer := chat_bot.automatic_answer(
-            message_text, region, zammad_chat.language.slug
-        ):
-            actions.append("automatic answer")
-            client.send_message(
-                zammad_chat.zammad_id,
-                answer,
-                False,
-                True,
-            )
     else:
-        actions.append("answer translation")
-        client.send_message(
-            zammad_chat.zammad_id,
-            chat_bot.automatic_translation(
-                message_text, region.default_language.slug, zammad_chat.language.slug
-            ),
-            False,
-            True,
+        actions.append("answer translation queued")
+        process_answer.apply_async(
+            args=[message_text, region.slug, webhook_message["ticket"]["id"]]
         )
     return JsonResponse(
         {

integreat_cms/api/v3/chat/utils/__init__.py

@@ -0,0 +1,3 @@
+"""
+Utils for the Integreat Chat
+"""

integreat_cms/api/v3/chat/utils/chat_bot.py

@@ -0,0 +1,111 @@
+"""
+Wrapper for the Chat Bot / LLM API
+"""
+
+from __future__ import annotations
+
+import requests
+from celery import shared_task
+from django.conf import settings
+
+from integreat_cms.cms.models import Region, UserChat
+from integreat_cms.cms.utils.content_translation_utils import (
+    get_public_translation_for_link,
+)
+
+from .zammad_api import ZammadChatAPI
+
+
+def format_message(response: dict) -> str:
+    """
+    Transform JSON into readable message
+    """
+    if "answer" not in response or not response["answer"]:
+        raise ValueError("Could not format message, no answer attribute in response")
+    if "sources" not in response or not response["sources"]:
+        return response["answer"]
+    sources = "".join(
+        [
+            f"<li><a href='{settings.WEBAPP_URL}{path}'>{title}</a></li>"
+            for path in response["sources"]
+            if (title := get_public_translation_for_link(settings.WEBAPP_URL + path))
+        ]
+    )
+    return f"{response['answer']}\n<ul>{sources}</ul>"
+
+
+def automatic_answer(message: str, region: Region, language_slug: str) -> str | None:
+    """
+    Get automatic answer to question
+    """
+    url = (
+        f"https://{settings.INTEGREAT_CHAT_BACK_END_DOMAIN}/chatanswers/extract_answer/"
+    )
+    body = {"message": message, "language": language_slug, "region": region.slug}
+    r = requests.post(url, json=body, timeout=120)
+    return format_message(r.json())
+
+
+def automatic_translation(
+    message: str, source_language_slug: str, target_language_slug: str
+) -> str:
+    """
+    Use LLM to translate message
+    """
+    url = f"https://{settings.INTEGREAT_CHAT_BACK_END_DOMAIN}/chatanswers/translate_message/"
+    body = {
+        "message": message,
+        "source_language": source_language_slug,
+        "target_language": target_language_slug,
+    }
+    response = requests.post(url, json=body, timeout=120).json()
+    if "status" in response and response["status"] == "success":
+        return response["translation"]
+    raise ValueError("Did not receive success response for translation request.")
+
+
+@shared_task
+def process_user_message(
+    message_text: str, region_slug: str, zammad_ticket_id: int
+) -> None:
+    """
+    Process the message from an Integreat App user
+    """
+    zammad_chat = UserChat.objects.get(zammad_id=zammad_ticket_id)
+    region = Region.objects.get(slug=region_slug)
+    client = ZammadChatAPI(region)
+    if translation := automatic_translation(
+        message_text, zammad_chat.language.slug, region.default_language.slug
+    ):
+        client.send_message(
+            zammad_chat.zammad_id,
+            translation,
+            True,
+            True,
+        )
+    if answer := automatic_answer(message_text, region, zammad_chat.language.slug):
+        client.send_message(
+            zammad_chat.zammad_id,
+            answer,
+            False,
+            True,
+        )
+
+
+@shared_task
+def process_answer(message_text: str, region_slug: str, zammad_ticket_id: int) -> None:
+    """
+    Process automatic or counselor answers
+    """
+    zammad_chat = UserChat.objects.get(zammad_id=zammad_ticket_id)
+    region = Region.objects.get(slug=region_slug)
+    client = ZammadChatAPI(region)
+    if translation := automatic_translation(
+        message_text, region.default_language.slug, zammad_chat.language.slug
+    ):
+        client.send_message(
+            zammad_chat.zammad_id,
+            translation,
+            False,
+            True,
+        )

integreat_cms/api/v3/chat/zammad_api.py → integreat_cms/api/v3/chat/utils/zammad_api.py

@@ -13,7 +13,7 @@
 from requests.exceptions import HTTPError
 from zammad_py import ZammadAPI
 
-from ....cms.models import AttachmentMap, Region, UserChat
+from integreat_cms.cms.models import AttachmentMap, Region, UserChat
 
 logger = logging.getLogger(__name__)
 
@@ -166,10 +166,20 @@ def get_messages(self, chat: UserChat) -> dict[str, dict | list[dict]]:
 
     # pylint: disable=method-hidden
     def send_message(
-        self, chat_id: int, message: str, internal: bool = False, auto: bool = False
+        self,
+        chat_id: int,
+        message: str,
+        internal: bool = False,
+        automatic_message: bool = False,
     ) -> dict:
         """
         Post a new message to the given ticket
+
+        param chat_id: Zammad ID of the chat
+        param message: The message body
+        param internal: keep the message internal in Zammad (do not show to user)
+        param automatic_message: sets title to "automatically generated message"
+        return: dict with Zammad article data
         """
         params = {
             "ticket_id": chat_id,
@@ -178,9 +188,11 @@ def send_message(
             "content_type": "text/html",
             "internal": internal,
             "subject": (
-                "automatically generated message" if auto else "app user message"
+                "automatically generated message"
+                if automatic_message
+                else "app user message"
             ),
-            "sender": "Customer" if not auto else "Agent",
+            "sender": "Customer" if not automatic_message else "Agent",
         }
         return self._parse_response(  # type: ignore[return-value]
             self._attempt_call(self.client.ticket_article.create, params=params)

integreat_cms/cms/constants/frequency.py

@@ -14,6 +14,7 @@
 
     from django.utils.functional import Promise
 
+# The frequencies must adhere to the icalendar naming: https://icalendar.org/iCalendar-RFC-5545/3-3-10-recurrence-rule.html
 
 #: Daily
 DAILY: Final = "DAILY"