Merge pull request #35 from mozilla/34-support-smartling-cat

Add support for Smartling CAT

README.md

@@ -58,6 +58,8 @@ integrates with the Smartling translation platform.
     }
     ```
 
+    ----
+
     If your project's locales do not match those in Smartling (e.g. `ro` in your
     project, `ro-RO` in Smartling), then you can provide a Wagtail locale ID to
     Smartling locale ID mapping via the `LOCALE_TO_SMARTLING_LOCALE` setting:
@@ -104,6 +106,8 @@ integrates with the Smartling translation platform.
 
     ```
 
+    ----
+
     If you need to customize the default Job description, you can specify a callable or a dotted path to a callable in
     the `JOB_DESCRIPTION_CALLBACK` setting:
 
@@ -123,6 +127,44 @@ integrates with the Smartling translation platform.
     The callback receives the default description string, the job `TranslationSource` instance, and the list of
     target `Translation`s. It expected to return string.
 
+    ----
+
+    If you want to pass a [Visual Context](https://help.smartling.com/hc/en-us/articles/360057484273--Overview-of-Visual-Context)
+    to Smartling after a Job is synced, you need to provide a way to get hold
+    of the appropriate URL for the page to use context. You provide this via
+    the `VISUAL_CONTEXT_CALLBACK` setting.
+
+    If this callback is defined, it will be used to send the visual context to Smartling.
+    This step happens just after the regular sync of a Job to Smartling and _only_ if
+    the callback is defined.
+
+    The callback must take the Job instance and return:
+
+    1. a URL for the page that shows the content used to generate that Job
+    2. the HTML of the page.
+
+    ```python
+    from wagtail_localize.models import Job
+
+    def get_visual_context(job: Job) -> tuple[str, str]:
+
+        # This assumes the page is live and visible. If the page is a draft, you
+        # will need a some custom work to expose the draft version of the page
+        page = job.translation_source.get_source_instance()
+        page_url = page.full_url
+
+        html = # code to render that page instance
+
+        return page_url, html
+    ```
+
+    Note that if the syncing of the visual context fails, this will break the
+    overall sync to Smartling, leaving an inconsistent state:
+    there'll be a Job created in Smartling that's awaiting approval, but Wagtail
+    will still think the job needs to be created. This, in turn, will mean we get
+    duplicate job errors on the retry. Therefore, it is essential you have log
+    handling set up to catch the `ERROR`-level alert that will happen at this point.
+
 4. Run migrations:
 
     ```sh

src/wagtail_localize_smartling/__init__.py

@@ -1,5 +1,5 @@
 default_app_config = "wagtail_localize_smartling.apps.WagtailLocalizeSmartlingAppConfig"
 
 
-VERSION = (0, 5, 0)
+VERSION = (0, 6, 0)
 __version__ = ".".join(map(str, VERSION))

src/wagtail_localize_smartling/api/client.py

@@ -30,6 +30,7 @@
 from . import types
 from .serializers import (
     AddFileToJobResponseSerializer,
+    AddVisualContextToJobSerializer,
     AuthenticateResponseSerializer,
     CreateJobResponseSerializer,
     GetJobDetailsResponseSerializer,
@@ -384,6 +385,103 @@ def add_file_to_job(self, *, job: "Job"):
             json=body,
         )
 
+    def add_html_context_to_job(self, *, job: "Job"):
+        """
+        To help with translation, Smartling supports the idea of a
+        "visual context" for translators, which effectively gives them
+        a real-time/WYSIWYG view of the page they are translating.
+
+        We push info about the context, then trigger its processing, via
+        a special combined-action API endpoint:
+        https://api-reference.smartling.com/#tag/Context/operation/uploadAndMatchVisualContext
+
+        As for how we get the info to send as a visual context, that is up to the
+        implementation that is using wagtail-localize-smartling to decide, via
+        the use of a configurable callback function - see `VISUAL_CONTEXT_CALLBACK`
+        in the settings or the README.
+
+        If the callback is defined, it will be used to generate the the visual
+        context to send to Smartling.
+
+        The callback must take the Job instance and return:
+
+        1. A full, absolute URL for the page that shows the content used
+           to generate that Job
+        2. The HTML of that same page
+
+        e.g.
+
+            from wagtail_localize.models import Job
+
+            def get_visual_context(job: Job) -> tuple[str, str]:
+
+                # This assumes the page is live and visible. If the page is a
+                # draft, you will need a some custom work to expose the draft
+                # version of the page
+
+                page = job.translation_source.get_source_instance()
+                page_url = page.full_url
+
+                html = # code to render that page instance
+
+                return page_url, html
+
+        """
+
+        if not (
+            visual_context_callback_fn := smartling_settings.VISUAL_CONTEXT_CALLBACK
+        ):
+            return
+
+        url, html = visual_context_callback_fn(job)
+
+        # data:
+        # `name` - url of the page the Job is for
+        # `matchparams` - config params for Smartling's string matching
+        # `content` - the HTML of the relevant Page for this Job, as bytes
+
+        data_payload: dict[str, Any] = {
+            "name": url,
+            "matchparams": {
+                "translationJobUids": [job.translation_job_uid],
+            },
+        }
+
+        # The file payload contains the rendered HTML of the page
+        # being translated. It needs to be send as multipart form
+        # data, so we turn the HTML string into a bytearray
+        # and pass it along with a filename based on the slug
+        # of the page
+
+        if isinstance(html, str):
+            html = bytearray(html, "utf-8")
+
+        filename = utils.get_filename_for_visual_context(url)
+
+        file_payload: dict[str, tuple[str, bytes, str]] = {
+            "content": (filename, html, "text/html"),
+        }
+
+        logger.info(
+            "Sending visual context to Smartling for Job %s for URL %s",
+            job.translation_job_uid,
+            url,
+        )
+
+        result = self._request(
+            method="POST",
+            path=f"/context-api/v2/projects/{quote(job.project.project_id)}/contexts/upload-and-match-async",
+            response_serializer_class=AddVisualContextToJobSerializer,
+            files=file_payload,
+            data=data_payload,
+        )
+
+        logger.info(
+            "Visual context sent. processUid returned: %s", result.get("processUid")
+        )
+
+        return result
+
     @contextmanager
     def download_translations(self, *, job: "Job") -> Generator[ZipFile, None, None]:
         # This is an unusual case where a successful response is a ZIP file,

src/wagtail_localize_smartling/api/serializers.py

@@ -249,3 +249,7 @@ class UploadFileResponseSerializer(ResponseSerializer):
 class AddFileToJobResponseSerializer(ResponseSerializer):
     failCount = serializers.IntegerField()
     successCount = serializers.IntegerField()
+
+
+class AddVisualContextToJobSerializer(ResponseSerializer):
+    processUid = serializers.CharField()

src/wagtail_localize_smartling/api/types.py

@@ -104,3 +104,7 @@ class SourceFileData(TypedDict):
 class GetJobDetailsResponseData(CreateJobResponseData):
     priority: int | None
     sourceFiles: list[SourceFileData]
+
+
+class AddVisualContextToJobResponseData(TypedDict):
+    processUid: str

src/wagtail_localize_smartling/settings.py

@@ -13,6 +13,8 @@
 if TYPE_CHECKING:
     from wagtail_localize.models import Translation, TranslationSource
 
+    from wagtail_localize_smartling.models import Job
+
 logger = logging.getLogger(__name__)
 
 
@@ -34,6 +36,7 @@ class SmartlingSettings:
     JOB_DESCRIPTION_CALLBACK: (
         Callable[[str, "TranslationSource", Iterable["Translation"]], str] | None
     ) = None
+    VISUAL_CONTEXT_CALLBACK: Callable[["Job"], tuple[str, str]] | None = None
 
 
 def _init_settings() -> SmartlingSettings:
@@ -147,6 +150,14 @@ def _init_settings() -> SmartlingSettings:
         if callable(func_or_path):
             settings_kwargs["JOB_DESCRIPTION_CALLBACK"] = func_or_path
 
+    if "VISUAL_CONTEXT_CALLBACK" in settings_dict:
+        func_or_path = settings_dict["VISUAL_CONTEXT_CALLBACK"]
+        if isinstance(func_or_path, str):
+            func_or_path = import_string(func_or_path)
+
+        if callable(func_or_path):
+            settings_kwargs["VISUAL_CONTEXT_CALLBACK"] = func_or_path
+
     return SmartlingSettings(**settings_kwargs)
 
 

src/wagtail_localize_smartling/sync.py

@@ -71,6 +71,8 @@ def _initial_sync(job: "Job") -> None:
     """
     For jobs that have never been synced before, create the job in Smartling and
     add the PO file from the TranslationSource.
+
+    Also add Visual Context for Smartling CAT, if a callback to get that is configured
     """
     logger.info("Performing initial sync for job %s", job)
 
@@ -108,6 +110,9 @@ def _initial_sync(job: "Job") -> None:
     # Add the PO file to the job using the previously-saved URI
     client.add_file_to_job(job=job)
 
+    # Add context to the job (if settings.VISUAL_CONTEXT_CALLBACK is defined)
+    client.add_html_context_to_job(job=job)
+
 
 def _sync(job: "Job") -> None:
     """

src/wagtail_localize_smartling/utils.py

@@ -1,7 +1,7 @@
 import hashlib
 
 from typing import TYPE_CHECKING
-from urllib.parse import quote, urljoin
+from urllib.parse import quote, urljoin, urlparse
 
 from wagtail.coreutils import (
     get_content_languages,
@@ -111,7 +111,6 @@ def get_wagtail_source_locale(project: "Project") -> Locale | None:
     return locale
 
 
-# TODO test
 def suggest_source_locale(project: "Project") -> tuple[str, str] | None:
     """
     Return a tuple of language code and label for a suggested Locale from
@@ -141,3 +140,23 @@ def compute_content_hash(pofile: "POFile") -> str:
         strings.append(f"{entry.msgctxt}: {entry.msgid}")
 
     return hashlib.sha256("".join(strings).encode()).hexdigest()
+
+
+def get_filename_for_visual_context(url: str, max_length: int = 256) -> str:
+    """
+    Turn the given url into a long sluglike HTML filename, based
+    on the hostname and the path
+    """
+    if not url:
+        return url
+
+    _parsed = urlparse(url)
+
+    _hostname = _parsed.hostname or ""
+    _path = _parsed.path or ""
+
+    head = "-".join(_hostname.split(".")).rstrip("-").lower()
+    tail = "-".join(_path.split("/")).rstrip("-").lower()
+    body = f"{head}{tail}"[: max_length - 5]
+
+    return f"{body}.html"

testapp/settings.py

@@ -9,10 +9,15 @@
 """
 
 import os
+import typing
 
 import dj_database_url
 
 
+if typing.TYPE_CHECKING:
+    from wagtail_localize_smartling.models import Job
+
+
 # Build paths inside the project like this: os.path.join(PROJECT_DIR, ...)
 PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 BASE_DIR = os.path.dirname(PROJECT_DIR)
@@ -215,3 +220,7 @@ def map_project_locale_to_smartling(locale: str) -> str:
 
 def job_description_callback(description: str, translation_source, translations) -> str:
     return "1337"
+
+
+def visual_context_callback(job: "Job") -> tuple[str, str]:
+    return "https://example.com/path/to/page/", "<html><body>test</body></html>"

tests/conftest.py

@@ -14,6 +14,7 @@
 
 from wagtail_localize_smartling.api.client import client
 from wagtail_localize_smartling.api.types import (
+    AddVisualContextToJobResponseData,
     AuthenticateResponseData,
     GetProjectDetailsResponseData,
     TargetLocaleData,
@@ -146,6 +147,56 @@ def smartling_project(responses, settings, smartling_auth):
     return Project.get_current()
 
 
+@pytest.fixture()
+def smartling_add_visual_context(responses, settings, smartling_auth):
+    # Mock API request for sending visual context
+    project_id = settings.WAGTAIL_LOCALIZE_SMARTLING["PROJECT_ID"]
+    responses.assert_all_requests_are_fired = False
+    responses.add(
+        method="POST",
+        url=f"https://api.smartling.com/context-api/v2/projects/{quote(project_id)}/contexts/upload-and-match-async",
+        body=json.dumps(
+            {
+                "response": {
+                    "code": "SUCCESS",
+                    "data": AddVisualContextToJobResponseData(
+                        processUid="dummy_process_uid",
+                    ),
+                },
+            }
+        ),
+    )
+
+    return "dummy_process_uid"
+
+
+@pytest.fixture()
+def smartling_add_visual_context__error_response(responses, settings, smartling_auth):
+    # Mock API request for sending visual context
+    project_id = settings.WAGTAIL_LOCALIZE_SMARTLING["PROJECT_ID"]
+    responses.assert_all_requests_are_fired = False
+    responses.add(
+        method="POST",
+        url=f"https://api.smartling.com/context-api/v2/projects/{quote(project_id)}/contexts/upload-and-match-async",
+        body=json.dumps(
+            {
+                "response": {
+                    "code": "VALIDATION_ERROR",
+                    "data": [
+                        {
+                            "key": "some key",
+                            "message": "some message",
+                            "details": "some details",
+                        }
+                    ],
+                },
+            }
+        ),
+    )
+
+    return "dummy_process_uid"
+
+
 @pytest.fixture
 def smartling_settings():
     """

tests/management_commands/test_sync_smartling.py

@@ -6,7 +6,7 @@
 from tests.factories import JobFactory
 
 
-@pytest.mark.skip()
+@pytest.mark.skip("WRITE ME")
 @pytest.mark.django_db()
 def test_sync_smartling(smartling_project):
     unsynced_job_page = InfoPageFactory()