From 1baf987df802f2986378427cc40c51feefe39eb3 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 16 Jul 2024 16:00:25 +0100 Subject: [PATCH] DOC: Update child emulator documentation --- docs/source/asking_questions.rst | 2 - docs/source/manifest.rst | 2 +- docs/source/testing_services.rst | 233 +++++++++++++---------- docs/source/troubleshooting_services.rst | 45 ++--- 4 files changed, 152 insertions(+), 130 deletions(-) diff --git a/docs/source/asking_questions.rst b/docs/source/asking_questions.rst index a26090e2e..e5113893a 100644 --- a/docs/source/asking_questions.rst +++ b/docs/source/asking_questions.rst @@ -123,7 +123,6 @@ access the event store and run: { "event": { "kind": "delivery_acknowledgement", - "datetime": "2024-03-06T15:44:18.156044" }, }, { @@ -156,7 +155,6 @@ access the event store and run: { "event": { "kind": "heartbeat", - "datetime": "2024-03-06T15:46:18.167424" }, }, { diff --git a/docs/source/manifest.rst b/docs/source/manifest.rst index 21e503479..0b299343e 100644 --- a/docs/source/manifest.rst +++ b/docs/source/manifest.rst @@ -83,7 +83,7 @@ Download all or a subset of datasets from a manifest. .. note:: - Datasets are downloaded to temporary directories if no paths are given. + Datasets are downloaded to a temporary directory if no paths are given. Further information diff --git a/docs/source/testing_services.rst b/docs/source/testing_services.rst index 996f089f4..491067cc8 100644 --- a/docs/source/testing_services.rst +++ b/docs/source/testing_services.rst @@ -25,110 +25,99 @@ The Child Emulator We've written a child emulator that takes a list of events and returns them to the parent for handling in the order given - without contacting the real child or using Pub/Sub. Any events a real child can produce are supported. :mod:`Child ` instances can be mocked like-for-like by -:mod:`ChildEmulator ` instances without the parent knowing. You can provide -the emulated events in python or via a JSON file. +:mod:`ChildEmulator ` instances without the parent knowing. -Message types +Event kinds ------------- -You can emulate any message type that your app (the parent) can handle. The table below shows what these are. - -+-----------------------+--------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ -| Message type | Number of events supported | Example | -+=======================+==================================================================================================+===========================================================================================================================+ -| ``log_record`` | Any number | {"type": "log_record": "log_record": {"msg": "Starting analysis."}} | -+-----------------------+--------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ -| ``monitor_message`` | Any number | {"type": "monitor_message": "data": '{"progress": "35%"}'} | -+-----------------------+--------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ -| ``exception`` | One | {"type": "exception", "exception_type": "ValueError", "exception_message": "x cannot be less than 10."} | -+-----------------------+--------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ -| ``result`` | One | {"type": "result", "output_values": {"my": "results"}, "output_manifest": None} | -+-----------------------+--------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ +You can emulate any event that your app (the parent) can handle. The table below shows what these are. + ++------------------------------+----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Event kind | Number of events supported | Example | ++==============================+============================+=================================================================================================================================================================================================+ +| ``delivery_acknowledgement`` | One | {"event": {"kind": "delivery_acknowledgement"}, "attributes": {"question_uuid": 79192e90-9022-4797-b6c7-82dc097dacdb, ...}} | ++------------------------------+----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``heartbeat`` | Any number | {"event": {"kind": "heartbeat"}, "attributes": {"question_uuid": 79192e90-9022-4797-b6c7-82dc097dacdb, ...} | ++------------------------------+----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``log_record`` | Any number | {"event": {"kind": "log_record": "log_record": {"msg": "Starting analysis."}}, "attributes": {"question_uuid": 79192e90-9022-4797-b6c7-82dc097dacdb, ...} | ++------------------------------+----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``monitor_message`` | Any number | {"event": {"kind": "monitor_message": "data": '{"progress": "35%"}'}, "attributes": {"question_uuid": 79192e90-9022-4797-b6c7-82dc097dacdb, ...} | ++------------------------------+----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``exception`` | One | {"event": {"kind": "exception", "exception_type": "ValueError", "exception_message": "x cannot be less than 10."}, "attributes": {"question_uuid": 79192e90-9022-4797-b6c7-82dc097dacdb, ...} | ++------------------------------+----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``result`` | One | {"event": {"kind": "result", "output_values": {"my": "results"}, "output_manifest": None}, "attributes": {"question_uuid": 79192e90-9022-4797-b6c7-82dc097dacdb, ...} | ++------------------------------+----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ **Notes** -- Message formats and contents are validated by :mod:`ChildEmulator ` -- The ``log_record`` key of a ``log_record`` message is any dictionary that the ``logging.makeLogRecord`` function can +- Event formats and contents must conform with the `service communication schema `_. +- Every event must be accompanied with the required event attributes +- The ``log_record`` key of a ``log_record`` event is any dictionary that the ``logging.makeLogRecord`` function can convert into a log record. -- The ``data`` key of a ``monitor_message`` message must be a JSON-serialised string -- Any events after a ``result`` or ``exception`` message won't be passed to the parent because execution of the child +- The ``data`` key of a ``monitor_message`` event must be a JSON-serialised string +- Any events after a ``result`` or ``exception`` event won't be passed to the parent because execution of the child emulator will have ended. -Instantiating a child emulator in python ----------------------------------------- +Instantiating a child emulator +------------------------------ .. code-block:: python events = [ { - "type": "log_record", - "log_record": {"msg": "Starting analysis."}, + { + "event": { + "kind": "log_record", + "log_record": {"msg": "Starting analysis."}, + ... # Left out for brevity. + }, + "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", + "retry_count": 0, + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "parent_question_uuid": "5776ad74-52a6-46f7-a526-90421d91b8b2", + "originator_question_uuid": "86dc55b2-4282-42bd-92d0-bd4991ae7356", + "parent": "octue/test-service:1.0.0", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", + "sender_type": "CHILD", + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" + }, + }, }, { - "type": "monitor_message", - "data": '{"progress": "35%"}', + "event": { + "kind": "monitor_message", + "data": '{"progress": "35%"}', + }, + "attributes": { + ... # Left out for brevity. + }, }, { - "type": "log_record", - "log_record": {"msg": "Finished analysis."}, + "event": { + "kind": "log_record", + "log_record": {"msg": "Finished analysis."}, + ... # Left out for brevity. + }, + "attributes": { + ... # Left out for brevity. + }, }, { - "type": "result", - "output_values": [1, 2, 3, 4, 5], - "output_manifest": None, - }, - ] - - child_emulator = ChildEmulator( - backend={"name": "GCPPubSubBackend", "project_name": "my-project"}, - events=events - ) - - def handle_monitor_message(message): - ... - - result, question_uuid = child_emulator.ask( - input_values={"hello": "world"}, - handle_monitor_message=handle_monitor_message, - ) - >>> {"output_values": [1, 2, 3, 4, 5], "output_manifest": None} - - -Instantiating a child emulator from a JSON file ------------------------------------------------ -You can provide a JSON file with either just events in or with events and some or all of the -:mod:`ChildEmulator ` constructor parameters. Here's an example JSON file -with just the events: - -.. code-block:: json - - { - "events": [ - { - "type": "log_record", - "log_record": {"msg": "Starting analysis."} - }, - { - "type": "log_record", - "log_record": {"msg": "Finished analysis."} + "event": { + "kind": "result", + "output_values": [1, 2, 3, 4, 5], }, - { - "type": "monitor_message", - "data": "{\"progress\": \"35%\"}" + "attributes": { + ... # Left out for brevity. }, - { - "type": "result", - "output_values": [1, 2, 3, 4, 5], - "output_manifest": null - } - ] - } - -You can then instantiate a child emulator from this in python: - -.. code-block:: python + }, + ] - child_emulator = ChildEmulator.from_file("path/to/emulated_child.json") + child_emulator = ChildEmulator(events) def handle_monitor_message(message): ... @@ -177,21 +166,40 @@ To emulate your children in tests, patch the :mod:`Child >> [300] + + analysis.output_manifest + >>> None + **Notes** @@ -208,9 +216,10 @@ change correspondingly (or at all). So, it's up to you to define a set of realis (the list of emulated events) to test your service. These are called **test fixtures**. .. note:: - Unlike a real child, the inputs given to the emulator and the outputs returned aren't validated against the schema in - the child's twine - this is because the twine is only available to the real child. This is ok - you're testing your - service, not the child. + Unlike a real child, the **inputs** given to the emulator aren't validated against the schema in the child's twine - + this is because the twine is only available to the real child. This is ok - you're testing your service, not the + child your service contacts. The events given to the emulator are still validated against the service communication + schema, though. You can create test fixtures manually or by using the ``Child.received_events`` property after questioning a real child. @@ -231,23 +240,36 @@ child. child.received_events >>> [ { - 'type': 'delivery_acknowledgement', - 'delivery_time': '2022-08-16 11:49:57.244263', + "event": { + 'kind': 'delivery_acknowledgement', + }, + "attributes": { + ... # Left out for brevity. + }, }, { - 'type': 'log_record', - 'log_record': { - 'msg': 'Finished analysis.', - 'args': None, - 'levelname': 'INFO', - ... + "event": { + 'kind': 'log_record', + 'log_record': { + 'msg': 'Finished analysis.', + 'args': None, + 'levelname': 'INFO', + ... # Left out for brevity. + }, + }, + "attributes": { + ... # Left out for brevity. }, }, { - 'type': 'result', - 'output_values': {"some": "results"}, - 'output_manifest': None, - } + "event": { + 'kind': 'result', + 'output_values': {"some": "results"}, + }, + "attributes": { + ... # Left out for brevity. + }, + }, ] You can then feed these into a child emulator to emulate one possible response of the child: @@ -258,8 +280,9 @@ You can then feed these into a child emulator to emulate one possible response o child_emulator = ChildEmulator(events=child.received_events) + result, question_uuid = child_emulator.ask(input_values=[1, 2, 3, 4]) - child_emulator.ask(input_values=[1, 2, 3, 4]) - >>> {"some": "results"}, "9cab579f-c486-4324-ac9b-96491d26266b" + result + >>> {"some": "results"} -You can also create test fixtures from :ref:`downloaded service crash diagnostics `. +You can also create test fixtures from :ref:`downloaded service crash diagnostics `. diff --git a/docs/source/troubleshooting_services.rst b/docs/source/troubleshooting_services.rst index f60ca5fde..3e77d9aab 100644 --- a/docs/source/troubleshooting_services.rst +++ b/docs/source/troubleshooting_services.rst @@ -2,15 +2,16 @@ Troubleshooting services ======================== -Crash diagnostics -================= -Services save the following data to the cloud if they crash while processing a question: +Diagnostics +=========== +Services save the following data to the cloud if they crash while processing a question (the default), or when they +finish processing a question successfully if diagnostics are permanently turned on (not the default): - Input values - Input manifest and datasets - Child configuration values - Child configuration manifest and datasets -- Inputs to and messages received in answer to each question the service asked its children (if it has any). These are +- Inputs to and events received in response to each question the service asked its children (if it has any). These are stored in the order the questions were asked. .. important:: @@ -19,24 +20,24 @@ Services save the following data to the cloud if they crash while processing a q configuration (:ref:`octue.yaml ` file) set to a Google Cloud Storage path. -Accessing crash diagnostics -=========================== -In the event of a crash, the service will upload the crash diagnostics and send the upload path to the parent as a log -message. A user with credentials to access this path can use the ``octue`` CLI to retrieve the crash diagnostics data: +Accessing diagnostics +===================== +If diagnostics are enabled, a service will upload the diagnostics and send the upload path to the parent as a log +message. A user with credentials to access this path can use the ``octue`` CLI to retrieve the diagnostics data: .. code-block:: shell - octue get-crash-diagnostics + octue get-diagnostics More information on the command: .. code-block:: - >>> octue get-crash-diagnostics -h + >>> octue get-diagnostics -h - Usage: octue get-crash-diagnostics [OPTIONS] CLOUD_PATH + Usage: octue get-diagnostics [OPTIONS] CLOUD_PATH - Download crash diagnostics for an analysis from the given directory in + Download diagnostics for an analysis from the given directory in Google Cloud Storage. The cloud path should end in the analysis ID. CLOUD_PATH: The path to the directory in Google Cloud Storage containing the @@ -46,16 +47,16 @@ More information on the command: --local-path DIRECTORY The path to a directory to store the directory of diagnostics data in. Defaults to the current working directory. - --download-datasets If provided, download any datasets from the crash + --download-datasets If provided, download any datasets from the diagnostics and update their paths in their manifests to the new local paths. -h, --help Show this message and exit. -.. _test_fixtures_from_crash_diagnostics: +.. _test_fixtures_from_diagnostics: -Creating test fixtures from crash diagnostics -============================================= -You can create test fixtures directly from crash diagnostics, allowing you to recreate the exact conditions that caused +Creating test fixtures from diagnostics +======================================= +You can create test fixtures directly from diagnostics, allowing you to recreate the exact conditions that caused your service to fail. .. code-block:: python @@ -63,7 +64,7 @@ your service to fail. from unittest.mock import patch from octue import Runner - from octue.utils.testing import load_test_fixture_from_crash_diagnostics + from octue.utils.testing import load_test_fixture_from_diagnostics ( @@ -72,7 +73,7 @@ your service to fail. input_values, input_manifest, child_emulators, - ) = load_test_fixture_from_crash_diagnostics(path="path/to/downloaded/crash/diagnostics") + ) = load_test_fixture_from_diagnostics(path="path/to/downloaded/diagnostics") # You can explicitly specify your children here as shown or # read the same information in from your app configuration file. @@ -108,9 +109,9 @@ your service to fail. analysis = runner.run(input_values=input_values, input_manifest=input_manifest) -Disabling crash diagnostics -=========================== -When asking a question to a child, parents can disable crash diagnostics upload in the child on a question-by-question +Disabling diagnostics +===================== +When asking a question to a child, parents can disable diagnostics upload in the child on a question-by-question basis by setting ``save_diagnostics`` to ``"SAVE_DIAGNOSTICS_OFF"`` in :mod:`Child.ask `. For example: