Merge pull request kobotoolbox#2633 from kobotoolbox/kobotoolbox/task…

…s#341-ssrf-protection

Updated SSRF Protect version, fixed tests using it

dependencies/pip/dev_requirements.txt

@@ -5,10 +5,9 @@
 #    make pip_compile
 #
 -e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest  # via -r dependencies/pip/requirements.in
--e git+https://github.com/st8st8/django-markitup@1370b43dc519b1441510a17330b8e980d29a7bf3#egg=django_markitup  # via -r dependencies/pip/requirements.in
--e git+https://github.com/kobotoolbox/django-request-cache@c240abdd660cc59f5f5280beee30b3e011332a34#egg=django_request_cache  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/kobotoolbox/formpack.git@a08b7626507c2ceff35b816f1e1e0845c85cce26#egg=formpack  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest  # via -r dependencies/pip/requirements.in
+-e git+https://github.com/kobotoolbox/ssrf-protect@9eec6c4aa37700c6e7ca90540a9407bd20acddb0#egg=ssrf_protect  # via -r dependencies/pip/requirements.in
 amqp==2.5.2               # via -r dependencies/pip/requirements.in, kombu
 anyjson==0.3.3            # via -r dependencies/pip/requirements.in
 argparse==1.4.0           # via unittest2
@@ -42,12 +41,14 @@ django-extensions==2.2.5  # via -r dependencies/pip/requirements.in
 django-js-asset==1.2.2    # via django-mptt
 django-loginas==0.3.6     # via -r dependencies/pip/requirements.in
 django-markdownx==2.0.28  # via -r dependencies/pip/requirements.in
+django-markitup==4.0.0    # via -r dependencies/pip/requirements.in
 django-mptt==0.10.0       # via -r dependencies/pip/requirements.in
 django-oauth-toolkit==1.2.0  # via -r dependencies/pip/requirements.in
 django-picklefield==2.0   # via django-constance
 django-private-storage==2.2.1  # via -r dependencies/pip/requirements.in
 django-redis-sessions==0.6.1  # via -r dependencies/pip/requirements.in
 django-registration-redux==2.6  # via -r dependencies/pip/requirements.in
+django-request-cache==1.2  # via -r dependencies/pip/requirements.in
 django-reversion==3.0.1   # via -r dependencies/pip/requirements.in
 django-storages==1.7.2    # via -r dependencies/pip/requirements.in
 django-taggit==1.1.0      # via -r dependencies/pip/requirements.in
@@ -113,7 +114,7 @@ requests==2.22.0          # via -r dependencies/pip/requirements.in, django-oaut
 responses==0.10.6         # via -r dependencies/pip/requirements.in
 s3transfer==0.2.1         # via boto3
 shortuuid==0.5.0          # via -r dependencies/pip/requirements.in
-six==1.13.0               # via bcrypt, cryptography, django-extensions, jsonschema, mock, packaging, prompt-toolkit, pynacl, pyopenssl, pyrsistent, python-dateutil, responses, traitlets, unittest2
+six==1.13.0               # via bcrypt, cryptography, django-extensions, jsonschema, mock, packaging, prompt-toolkit, pynacl, pyopenssl, pyrsistent, python-dateutil, responses, ssrf-protect, traitlets, unittest2
 sqlparse==0.3.0           # via -r dependencies/pip/requirements.in, django, django-debug-toolbar
 static3==0.7.0            # via -r dependencies/pip/requirements.in, dj-static
 statistics==1.0.3.5       # via formpack

dependencies/pip/external_services.txt

@@ -5,10 +5,9 @@
 #    make pip_compile
 #
 -e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest  # via -r dependencies/pip/requirements.in
--e git+https://github.com/st8st8/django-markitup@1370b43dc519b1441510a17330b8e980d29a7bf3#egg=django_markitup  # via -r dependencies/pip/requirements.in
--e git+https://github.com/kobotoolbox/django-request-cache@c240abdd660cc59f5f5280beee30b3e011332a34#egg=django_request_cache  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/kobotoolbox/formpack.git@a08b7626507c2ceff35b816f1e1e0845c85cce26#egg=formpack  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest  # via -r dependencies/pip/requirements.in
+-e git+https://github.com/kobotoolbox/ssrf-protect@9eec6c4aa37700c6e7ca90540a9407bd20acddb0#egg=ssrf_protect  # via -r dependencies/pip/requirements.in
 amqp==2.5.2               # via -r dependencies/pip/requirements.in, kombu
 anyjson==0.3.3            # via -r dependencies/pip/requirements.in
 argparse==1.4.0           # via unittest2
@@ -38,12 +37,14 @@ django-extensions==2.2.5  # via -r dependencies/pip/requirements.in
 django-js-asset==1.2.2    # via django-mptt
 django-loginas==0.3.6     # via -r dependencies/pip/requirements.in
 django-markdownx==2.0.28  # via -r dependencies/pip/requirements.in
+django-markitup==4.0.0    # via -r dependencies/pip/requirements.in
 django-mptt==0.10.0       # via -r dependencies/pip/requirements.in
 django-oauth-toolkit==1.2.0  # via -r dependencies/pip/requirements.in
 django-picklefield==2.0   # via django-constance
 django-private-storage==2.2.1  # via -r dependencies/pip/requirements.in
 django-redis-sessions==0.6.1  # via -r dependencies/pip/requirements.in
 django-registration-redux==2.6  # via -r dependencies/pip/requirements.in
+django-request-cache==1.2  # via -r dependencies/pip/requirements.in
 django-reversion==3.0.1   # via -r dependencies/pip/requirements.in
 django-storages==1.7.2    # via -r dependencies/pip/requirements.in
 django-taggit==1.1.0      # via -r dependencies/pip/requirements.in
@@ -90,7 +91,7 @@ requests==2.22.0          # via -r dependencies/pip/requirements.in, django-oaut
 responses==0.10.6         # via -r dependencies/pip/requirements.in
 s3transfer==0.2.1         # via boto3
 shortuuid==0.5.0          # via -r dependencies/pip/requirements.in
-six==1.13.0               # via cryptography, django-extensions, jsonschema, pyopenssl, pyrsistent, python-dateutil, responses, transifex-client, unittest2
+six==1.13.0               # via cryptography, django-extensions, jsonschema, pyopenssl, pyrsistent, python-dateutil, responses, ssrf-protect, transifex-client, unittest2
 sqlparse==0.3.0           # via -r dependencies/pip/requirements.in, django, django-debug-toolbar
 static3==0.7.0            # via -r dependencies/pip/requirements.in, dj-static
 statistics==1.0.3.5       # via formpack

dependencies/pip/requirements.in

@@ -9,11 +9,8 @@
 -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest
 -e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest
 
-# TODO: Get `django-markitup` from PyPI again once it supports Django 2.1
--e git+https://github.com/st8st8/django-markitup@1370b43dc519b1441510a17330b8e980d29a7bf3#egg=django_markitup
-
-# TODO²: Get `django-request-cache` from PyPI again once https://github.com/anexia-it/django-request-cache/pull/6 is merged
--e git+https://github.com/kobotoolbox/django-request-cache@c240abdd660cc59f5f5280beee30b3e011332a34#egg=django_request_cache
+# ssrf protect
+-e git+https://github.com/kobotoolbox/ssrf-protect@9eec6c4aa37700c6e7ca90540a9407bd20acddb0#egg=ssrf_protect
 
 # Regular PyPI packages
 Django>=2.2,<2.3
@@ -40,7 +37,7 @@ django-amazon-ses
 django-webpack-loader
 django-loginas
 django-markdownx
-# django-markitup # Not compatible with Django 2.0+ (see above ToDo²)
+django-markitup
 django-mptt
 django-reversion<3.0.2 # Migration issue with 3.0.2
 django-taggit
@@ -49,6 +46,7 @@ django-private-storage
 djangorestframework
 djangorestframework-xml
 django-redis-sessions
+django-request-cache
 drf-extensions
 future
 geojson-rewind
@@ -78,6 +76,3 @@ XlsxWriter
 pyopenssl
 ndg-httpsclient
 pyasn1
-
-# Cache-Request
-# django-request-cache # Not compatible with Django 2.1+ (see above ToDo³)

dependencies/pip/requirements.txt

@@ -5,10 +5,9 @@
 #    make pip_compile
 #
 -e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest  # via -r dependencies/pip/requirements.in
--e git+https://github.com/st8st8/django-markitup@1370b43dc519b1441510a17330b8e980d29a7bf3#egg=django_markitup  # via -r dependencies/pip/requirements.in
--e git+https://github.com/kobotoolbox/django-request-cache@c240abdd660cc59f5f5280beee30b3e011332a34#egg=django_request_cache  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/kobotoolbox/formpack.git@a08b7626507c2ceff35b816f1e1e0845c85cce26#egg=formpack  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest  # via -r dependencies/pip/requirements.in
+-e git+https://github.com/kobotoolbox/ssrf-protect@9eec6c4aa37700c6e7ca90540a9407bd20acddb0#egg=ssrf_protect  # via -r dependencies/pip/requirements.in
 amqp==2.5.2               # via -r dependencies/pip/requirements.in, kombu
 anyjson==0.3.3            # via -r dependencies/pip/requirements.in
 argparse==1.4.0           # via unittest2
@@ -38,12 +37,14 @@ django-extensions==2.2.5  # via -r dependencies/pip/requirements.in
 django-js-asset==1.2.2    # via django-mptt
 django-loginas==0.3.6     # via -r dependencies/pip/requirements.in
 django-markdownx==2.0.28  # via -r dependencies/pip/requirements.in
+django-markitup==4.0.0    # via -r dependencies/pip/requirements.in
 django-mptt==0.10.0       # via -r dependencies/pip/requirements.in
 django-oauth-toolkit==1.2.0  # via -r dependencies/pip/requirements.in
 django-picklefield==2.0   # via django-constance
 django-private-storage==2.2.1  # via -r dependencies/pip/requirements.in
 django-redis-sessions==0.6.1  # via -r dependencies/pip/requirements.in
 django-registration-redux==2.6  # via -r dependencies/pip/requirements.in
+django-request-cache==1.2  # via -r dependencies/pip/requirements.in
 django-reversion==3.0.1   # via -r dependencies/pip/requirements.in
 django-storages==1.7.2    # via -r dependencies/pip/requirements.in
 django-taggit==1.1.0      # via -r dependencies/pip/requirements.in
@@ -89,7 +90,7 @@ requests==2.22.0          # via -r dependencies/pip/requirements.in, django-oaut
 responses==0.10.6         # via -r dependencies/pip/requirements.in
 s3transfer==0.2.1         # via boto3
 shortuuid==0.5.0          # via -r dependencies/pip/requirements.in
-six==1.13.0               # via cryptography, django-extensions, jsonschema, pyopenssl, pyrsistent, python-dateutil, responses, unittest2
+six==1.13.0               # via cryptography, django-extensions, jsonschema, pyopenssl, pyrsistent, python-dateutil, responses, ssrf-protect, unittest2
 sqlparse==0.3.0           # via -r dependencies/pip/requirements.in, django, django-debug-toolbar
 static3==0.7.0            # via -r dependencies/pip/requirements.in, dj-static
 statistics==1.0.3.5       # via formpack

kobo/apps/hook/models/service_definition_interface.py

@@ -6,6 +6,7 @@
 
 import constance
 import requests
+from ssrf_protect.ssrf_protect import SSRFProtect, SSRFProtectException
 
 from kpi.utils.log import logging
 from .hook import Hook
@@ -100,6 +101,8 @@ def send(self):
                         "auth": (self._hook.settings.get("username"),
                                  self._hook.settings.get("password"))
                     })
+
+                SSRFProtect.validate(self._hook.endpoint)
                 response = requests.post(self._hook.endpoint, timeout=30, **request_kwargs)
                 response.raise_for_status()
                 self.save_log(response.status_code, response.text, True)
@@ -113,7 +116,15 @@ def send(self):
                     text = response.text
                     status_code = response.status_code
                 self.save_log(status_code, text)
-
+            except SSRFProtectException as e:
+                logging.error("service_json.ServiceDefinition.send - "
+                              "Hook #{} - Data #{} - {}".format(self._hook.uid,
+                                                                self._instance_id,
+                                                                str(e)),
+                              exc_info=True)
+                self.save_log(
+                    KOBO_INTERNAL_ERROR_STATUS_CODE,
+                    f'{self._hook.endpoint} is not allowed')
             except Exception as e:
                 logging.error("service_json.ServiceDefinition.send - "
                               "Hook #{} - Data #{} - {}".format(self._hook.uid,

kobo/apps/hook/tests/hook_test_case.py

@@ -4,6 +4,7 @@
 import responses
 from django.conf import settings
 from django.urls import reverse
+from ipaddress import ip_address
 from rest_framework import status
 
 from kpi.constants import INSTANCE_FORMAT_TYPE_JSON, INSTANCE_FORMAT_TYPE_XML
@@ -13,6 +14,13 @@
 from ..models import HookLog, Hook
 
 
+class MockSSRFProtect(object):
+
+    @staticmethod
+    def _get_ip_address(url):
+        return ip_address('1.2.3.4')
+
+
 class HookTestCase(KpiTestCase):
 
     def setUp(self):

kobo/apps/hook/tests/test_api_hook.py

@@ -5,6 +5,7 @@
 import responses
 from django.contrib.auth.models import User
 from django.urls import reverse
+from mock import patch
 from rest_framework import status
 
 from kobo.apps.hook.constants import SUBMISSION_PLACEHOLDER
@@ -14,7 +15,7 @@
     PERM_VIEW_SUBMISSIONS,
     PERM_CHANGE_ASSET
 )
-from .hook_test_case import HookTestCase
+from .hook_test_case import HookTestCase, MockSSRFProtect
 
 
 class ApiHookTestCase(HookTestCase):
@@ -49,6 +50,8 @@ def test_anonymous_access(self):
     def test_create_hook(self):
         self._create_hook()
 
+    @patch('ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address',
+           new=MockSSRFProtect._get_ip_address)
     @responses.activate
     def test_data_submission(self):
         # Create first hook
@@ -197,6 +200,8 @@ def test_partial_update_hook(self):
         self.assertFalse(hook.active)
         self.assertEqual(hook.name, "some disabled external service")
 
+    @patch('ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address',
+           new=MockSSRFProtect._get_ip_address)
     @responses.activate
     def test_send_and_retry(self):
 
@@ -223,6 +228,8 @@ def test_send_and_retry(self):
         response = self.client.get(detail_url, format=INSTANCE_FORMAT_TYPE_JSON)
         self.assertEqual(response.data.get("tries"), 2)
 
+    @patch('ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address',
+           new=MockSSRFProtect._get_ip_address)
     @responses.activate
     def test_payload_template(self):
 

kobo/apps/hook/tests/test_email.py

@@ -5,8 +5,9 @@
 from django.template.loader import get_template
 from django.utils import translation, dateparse
 from django_celery_beat.models import PeriodicTask
+from mock import patch
 
-from .hook_test_case import HookTestCase
+from .hook_test_case import HookTestCase, MockSSRFProtect
 from ..tasks import failures_reports
 
 
@@ -19,6 +20,8 @@ def _create_periodic_task(self):
                                      task=beat_schedule.get("task"))
         periodic_task.save()
 
+    @patch('ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address',
+           new=MockSSRFProtect._get_ip_address)
     @responses.activate
     def test_notifications(self):
         self._create_periodic_task()