automate spam detection

.dockerignore

@@ -1,3 +1,6 @@
+.direnv/
+.tool-versions
+.envrc
 .git
 .yarn/cache
 .yarn/install-state.gz

.gitignore

@@ -180,3 +180,7 @@ vignettes/*.pdf
 
 # End of https://www.toptal.com/developers/gitignore/api/r
 
+# asdf & direnv
+.direnv/
+.tool-versions
+.envrc

Makefile

@@ -12,7 +12,9 @@ SECRETS_DIR=${BUILD_DIR}/secrets
 DB_PASSWORD_PATH=${SECRETS_DIR}/db_password
 PGPASS_PATH=${SECRETS_DIR}/.pgpass
 SECRET_KEY_PATH=${SECRETS_DIR}/django_secret_key
-EXT_SECRETS=hcaptcha_secret github_client_secret orcid_client_secret discourse_api_key discourse_sso_secret mail_api_key datacite_api_password
+EXT_SECRETS=hcaptcha_secret github_client_secret orcid_client_secret discourse_api_key discourse_sso_secret mail_api_key datacite_api_password \
+			llm_spam_check_api_key llm_spam_check_jetstream_os_application_credential_secret llm_spam_check_jetstream_os_application_credential_id
+
 GENERATED_SECRETS=$(DB_PASSWORD_PATH) $(PGPASS_PATH) $(SECRET_KEY_PATH)
 
 ENVREPLACE := deploy/scripts/envreplace

base.yml

@@ -69,6 +69,9 @@ services:
       - orcid_client_secret
       - hcaptcha_secret
       - mail_api_key
+      - llm_spam_check_api_key
+      - llm_spam_check_jetstream_os_application_credential_secret
+      - llm_spam_check_jetstream_os_application_credential_id
     volumes:
       - ./deploy/elasticsearch.conf.d:/etc/elasticsearch
       - ./docker/shared:/shared
@@ -104,7 +107,12 @@ secrets:
     file: ./build/secrets/mail_api_key
   orcid_client_secret:
     file: ./build/secrets/orcid_client_secret
-
+  llm_spam_check_api_key:
+    file: ./build/secrets/llm_spam_check_api_key
+  llm_spam_check_jetstream_os_application_credential_secret:
+    file: ./build/secrets/llm_spam_check_jetstream_os_application_credential_secret
+  llm_spam_check_jetstream_os_application_credential_id:
+    file: ./build/secrets/llm_spam_check_jetstream_os_application_credential_id
 volumes:
   esdata:
     driver: local

deploy/conf/.env.template

@@ -45,3 +45,7 @@ DATACITE_DRY_RUN="true" # allowed values: "true" or "false"
 TEST_USER_ID=10000000
 TEST_USERNAME=__test_user__
 TEST_BASIC_AUTH_PASSWORD=
+
+# spam-check on jetstream2
+LLM_SPAM_CHECK_API_URL=
+LLM_SPAM_CHECK_JETSTREAM_SERVER_ID=

django/core/mixins.py

@@ -7,12 +7,12 @@
 from django.shortcuts import redirect
 from django.utils import timezone
 from rest_framework import serializers
+from rest_framework.decorators import action
 from rest_framework.exceptions import NotFound
 from rest_framework.response import Response
-from rest_framework.decorators import action
 
 from .models import SpamModeration
-from .permissions import ViewRestrictedObjectPermissions, ModeratorPermissions
+from .permissions import ModeratorPermissions, ViewRestrictedObjectPermissions
 
 logger = logging.getLogger(__name__)
 
@@ -249,11 +249,11 @@ class SpamCatcherViewSetMixin:
 
     def perform_create(self, serializer: serializers.Serializer):
         super().perform_create(serializer)
-        self.handle_spam_detection(serializer)
+        self.create_or_update_spam_moderation_object(serializer)
 
     def perform_update(self, serializer):
         super().perform_update(serializer)
-        self.handle_spam_detection(serializer)
+        self.create_or_update_spam_moderation_object(serializer)
 
     def _validate_content_object(self, instance):
         # make sure that the instance has a spam_moderation attribute as well as the
@@ -262,14 +262,19 @@ def _validate_content_object(self, instance):
             "spam_moderation",
             "is_marked_spam",
             "get_absolute_url",
-            "title",
         ]
         for field in required_fields:
             if not hasattr(instance, field):
                 raise ValueError(
                     f"instance {instance} does not have a {field} attribute"
                 )
 
+        # Ensure either 'title' () or 'username' (for MemberProfile) is present
+        if not (hasattr(instance, "title") or hasattr(instance, "username")):
+            raise ValueError(
+                f"instance {instance} must have either a 'title' or a 'username' attribute"
+            )
+
     @action(detail=True, methods=["post"], permission_classes=[ModeratorPermissions])
     def mark_spam(self, request, **kwargs):
         instance = self.get_object()
@@ -294,28 +299,43 @@ def mark_spam(self, request, **kwargs):
             spam_moderation.save()
         return redirect(instance.get_list_url())
 
-    def handle_spam_detection(self, serializer: serializers.Serializer):
-        if "spam_context" in serializer.context:
-            try:
-                self._validate_content_object(serializer.instance)
-                self._record_spam(
-                    serializer.instance, serializer.context["spam_context"]
-                )
-            except ValueError as e:
-                logger.warning("Cannot flag %s as spam: %s", serializer.instance, e)
+    def create_or_update_spam_moderation_object(
+        self, serializer: serializers.Serializer
+    ):
+        try:
+            self._validate_content_object(serializer.instance)
+            self._create_or_update_spam_moderation_object(
+                serializer.instance,
+                (
+                    serializer.context["spam_context"]
+                    if "spam_context" in serializer.context
+                    else None
+                ),
+            )
+        except ValueError as e:
+            logger.warning("Cannot flag %s as spam: %s", serializer.instance, e)
 
-    def _record_spam(self, instance, spam_context: dict):
+    def _create_or_update_spam_moderation_object(
+        self, instance, spam_context: dict = None
+    ):
         content_type = ContentType.objects.get_for_model(type(instance))
-        # SpamModeration updates the content instance on save
-        spam_moderation, created = SpamModeration.objects.get_or_create(
+        default_status = (
+            SpamModeration.Status.SPAM_LIKELY
+            if spam_context
+            else SpamModeration.Status.SCHEDULED_FOR_CHECK
+        )
+        default_spam_moderation = {
+            "status": default_status,
+            "detection_method": (
+                spam_context.get("detection_method", "") if spam_context else ""
+            ),
+            "detection_details": (
+                spam_context.get("detection_details", "") if spam_context else ""
+            ),
+        }
+
+        SpamModeration.objects.update_or_create(
             content_type=content_type,
             object_id=instance.id,
-            defaults={
-                "status": SpamModeration.Status.UNREVIEWED,
-                "detection_method": spam_context["detection_method"],
-                "detection_details": spam_context["detection_details"],
-            },
+            defaults=default_spam_moderation,
         )
-        if not created:
-            spam_moderation.status = SpamModeration.Status.UNREVIEWED
-            spam_moderation.save()

django/core/models.py

@@ -8,9 +8,9 @@
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.contrib.auth.models import Group, User
-from django.contrib.postgres.fields import ArrayField
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
+from django.contrib.postgres.fields import ArrayField
 from django.db import models, transaction
 from django.urls import reverse
 from django.utils import timezone
@@ -134,13 +134,15 @@ def deploy_environment(self):
 
 class SpamModeration(models.Model):
     class Status(models.TextChoices):
-        UNREVIEWED = "unreviewed", _("Unreviewed")
         SPAM = "spam", _("Confirmed spam")
         NOT_SPAM = "not_spam", _("Confirmed not spam")
+        SCHEDULED_FOR_CHECK = "scheduled_for_check", _("Scheduled for check by LLM")
+        SPAM_LIKELY = "spam_likely", _("Automatically marked as spam")
+        NOT_SPAM_LIKELY = "not_spam_likely", _("Automatically marked as not spam")
 
     status = models.CharField(
         choices=Status.choices,
-        default=Status.UNREVIEWED,
+        default=Status.SCHEDULED_FOR_CHECK,
         max_length=32,
     )
     content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
@@ -166,6 +168,16 @@ class Status(models.TextChoices):
         blank=True,
     )
 
+    # detection_details is a JSON field
+    def mark_as_spam_by_llm(self, status: Status, detection_details=None):
+        logger.info("Marking %s as %s by LLM", self, status)
+        self.reviewer = None
+        self.status = status
+        self.detection_method = "LLM"
+        if detection_details:
+            self.detection_details = detection_details
+        self.save()
+
     def mark_not_spam(self, reviewer: User, detection_details=None):
         logger.info("user %s marking %s as not spam", reviewer, self)
         self.status = self.Status.NOT_SPAM
@@ -182,7 +194,10 @@ def update_related_object(self):
         related_object = self.content_object
         if hasattr(related_object, "is_marked_spam"):
             related_object.spam_moderation = self
-            related_object.is_marked_spam = self.status != self.Status.NOT_SPAM
+            related_object.is_marked_spam = self.status in {
+                self.Status.SPAM,
+                self.Status.SPAM_LIKELY,
+            }
             related_object.save()
 
     def __str__(self):
@@ -340,7 +355,7 @@ def find_users_with_email(self, candidate_email, exclude_user=None):
 
 @add_to_comses_permission_whitelist
 @register_snippet
-class MemberProfile(index.Indexed, ClusterableModel):
+class MemberProfile(index.Indexed, ModeratedContent, ClusterableModel):
     """
     Contains additional comses.net information, possibly linked to a CoMSES Member / site account
     """

django/core/settings/defaults.py

@@ -540,6 +540,20 @@ def is_test(self):
 DISCOURSE_API_KEY = read_secret("discourse_api_key", "unconfigured")
 DISCOURSE_API_USERNAME = os.getenv("DISCOURSE_API_USERNAME", "unconfigured")
 
+
+LLM_SPAM_CHECK_API_URL = os.getenv("LLM_SPAM_CHECK_API_URL", "unconfigured")
+LLM_SPAM_CHECK_API_KEY = (
+    read_secret("llm_spam_check_api_key", "unconfigured") or "unconfigured"
+)
+
+LLM_SPAM_CHECK_JETSTREAM_SERVER_ID = os.getenv("LLM_SPAM_CHECK_JETSTREAM_SERVER_ID", "")
+LLM_SPAM_CHECK_JETSTREAM_OS_APPLICATION_CREDENTIAL_SECRET = read_secret(
+    "llm_spam_check_jetstream_os_application_credential_secret", "unconfigured"
+)
+LLM_SPAM_CHECK_JETSTREAM_OS_APPLICATION_CREDENTIAL_ID = read_secret(
+    "llm_spam_check_jetstream_os_application_credential_id", "unconfigured"
+)
+
 # https://docs.djangoproject.com/en/4.2/ref/settings/#templates
 TEMPLATES = [
     {