Merge pull request #187 from andreytaboola/master

Add support of keeping the local context of the python scripts and notebooks by preserving working directory

.circleci/config.yml

@@ -220,7 +220,7 @@ jobs:
       PYTHON_VERSION: "3_6"
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts/3_6
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_6
-      VERSION: 0.6.2
+      VERSION: 0.6.3
       PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases
       YARN_STATIC_DIR: notebooker/web/static/
       IMAGE_NAME: mangroup/notebooker
@@ -236,7 +236,7 @@ jobs:
     environment:
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts/3_7
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_7
-      VERSION: 0.6.2
+      VERSION: 0.6.3
       PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases
       YARN_STATIC_DIR: notebooker/web/static/
       IMAGE_NAME: mangroup/notebooker
@@ -250,7 +250,7 @@ jobs:
     environment:
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts/3_8
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_8
-      VERSION: 0.6.2
+      VERSION: 0.6.3
       PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases
       YARN_STATIC_DIR: notebooker/web/static/
       IMAGE_NAME: mangroup/notebooker
@@ -264,7 +264,7 @@ jobs:
     environment:
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts/3_11
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_11
-      VERSION: 0.6.2
+      VERSION: 0.6.3
       PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases
       YARN_STATIC_DIR: notebooker/web/static/
       IMAGE_NAME: mangroup/notebooker

CHANGELOG.md

@@ -1,3 +1,7 @@
+0.6.3 (2024-07-11)
+------------------
+* Feature: Flag to preserve original working directory when running notebooks to make local imports and relative paths work.
+
 0.6.2 (2024-05-02)
 ------------------
 * Bugfix: Folders with spaces in their names are now correctly handled in the webapp.

docs/conf.py

@@ -23,7 +23,7 @@
 author = "Man Group Quant Tech"
 
 # The full version, including alpha/beta/rc tags
-release = "0.6.2"
+release = "0.6.3"
 
 
 # -- General configuration ---------------------------------------------------

notebooker/_entrypoints.py

@@ -64,6 +64,12 @@ def filesystem_default_value(dirname):
     is_flag=True,
     help="If selected, notebooker will not try to pull the latest version of python templates from git.",
 )
+@click.option(
+    "--execute-at-origin",
+    default=False,
+    is_flag=True,
+    help="If selected, notebooker set current working directory to absolute path of the notebook to keep it local context available",
+)
 @click.option(
     "--default-mailfrom", default=DEFAULT_MAILFROM_ADDRESS, help="Set a new value for the default mailfrom setting."
 )
@@ -84,6 +90,7 @@ def base_notebooker(
     py_template_base_dir,
     py_template_subdir,
     notebooker_disable_git,
+    execute_at_origin,
     default_mailfrom,
     running_timeout,
     serializer_cls,
@@ -98,6 +105,7 @@ def base_notebooker(
         PY_TEMPLATE_BASE_DIR=py_template_base_dir,
         PY_TEMPLATE_SUBDIR=py_template_subdir,
         NOTEBOOKER_DISABLE_GIT=notebooker_disable_git,
+        EXECUTE_AT_ORIGIN=execute_at_origin,
         DEFAULT_MAILFROM=default_mailfrom,
         RUNNING_TIMEOUT=running_timeout,
     )

notebooker/execute_notebook.py

@@ -48,6 +48,7 @@ def _run_checks(
     email_subject: Optional[str] = "",
     prepare_only: Optional[bool] = False,
     notebooker_disable_git: bool = False,
+    execute_at_origin: bool = False,
     py_template_base_dir: str = "",
     py_template_subdir: str = "",
     scheduler_job_id: Optional[str] = None,
@@ -82,6 +83,10 @@ def _run_checks(
         Comma-separated email addresses to send on error.
     prepare_only : `Optional[bool]`
         Internal usage. Whether we want to do everything apart from executing the notebook.
+    notebooker_disable_git : `bool`
+        Whether to disable git pulling of the notebook templates.
+    execute_at_origin : `bool`
+        Whether to execute the notebook at the original path of the template notebook.
     scheduler_job_id : `Optional[str]`
         If available, it will be part of the Error or Completed run report.
     mailfrom : `Optional[str]`
@@ -111,8 +116,18 @@ def _run_checks(
     ipynb_executed_path = os.path.join(output_dir, output_ipynb)
 
     logger.info("Executing notebook at {} using parameters {} --> {}".format(ipynb_raw_path, overrides, output_ipynb))
+    working_dir = None
+    if execute_at_origin:
+        working_dir = os.path.dirname(os.path.join(py_template_dir, template_name))
+        logger.info("Setting working directory for execution {}".format(working_dir))
+
     pm.execute_notebook(
-        ipynb_raw_path, ipynb_executed_path, parameters=overrides, log_output=True, prepare_only=prepare_only
+        ipynb_raw_path,
+        ipynb_executed_path,
+        parameters=overrides,
+        log_output=True,
+        prepare_only=prepare_only,
+        cwd=working_dir,
     )
     with open(ipynb_executed_path, "r") as f:
         raw_executed_ipynb = f.read()
@@ -163,6 +178,7 @@ def run_report(
     hide_code=False,
     prepare_only=False,
     notebooker_disable_git=False,
+    execute_at_origin=False,
     py_template_base_dir="",
     py_template_subdir="",
     scheduler_job_id=None,
@@ -200,6 +216,7 @@ def run_report(
             hide_code=hide_code,
             prepare_only=prepare_only,
             notebooker_disable_git=notebooker_disable_git,
+            execute_at_origin=execute_at_origin,
             py_template_base_dir=py_template_base_dir,
             py_template_subdir=py_template_subdir,
             scheduler_job_id=scheduler_job_id,
@@ -353,6 +370,7 @@ def execute_notebook_entrypoint(
     output_dir, template_dir, _ = initialise_base_dirs(output_dir=config.OUTPUT_DIR, template_dir=config.TEMPLATE_DIR)
     all_overrides = _get_overrides(overrides_as_json, iterate_override_values_of)
     notebooker_disable_git = config.NOTEBOOKER_DISABLE_GIT
+    execute_at_origin = config.EXECUTE_AT_ORIGIN
     py_template_base_dir = config.PY_TEMPLATE_BASE_DIR
     py_template_subdir = config.PY_TEMPLATE_SUBDIR
 
@@ -376,6 +394,7 @@ def execute_notebook_entrypoint(
     logger.info("prepare_notebook_only = %s", prepare_notebook_only)
     logger.info("scheduler job id = %s", scheduler_job_id)
     logger.info("notebooker_disable_git = %s", notebooker_disable_git)
+    logger.info("execute_at_origin = %s", execute_at_origin)
     logger.info("py_template_base_dir = %s", py_template_base_dir)
     logger.info("py_template_subdir = %s", py_template_subdir)
     logger.info("serializer_cls = %s", config.SERIALIZER_CLS)
@@ -402,6 +421,7 @@ def execute_notebook_entrypoint(
             hide_code=hide_code,
             prepare_only=prepare_notebook_only,
             notebooker_disable_git=notebooker_disable_git,
+            execute_at_origin=execute_at_origin,
             py_template_base_dir=py_template_base_dir,
             py_template_subdir=py_template_subdir,
             scheduler_job_id=scheduler_job_id,
@@ -532,6 +552,7 @@ def run_report_in_subprocess(
             base_config.DEFAULT_MAILFROM,
         ]
         + (["--notebooker-disable-git"] if base_config.NOTEBOOKER_DISABLE_GIT else [])
+        + (["--execute-at-origin"] if base_config.EXECUTE_AT_ORIGIN else [])
         + ["--serializer-cls", result_serializer.__class__.__name__]
         + result_serializer.serializer_args_to_cmdline_args()
         + [

notebooker/settings.py

@@ -25,6 +25,9 @@ class BaseConfig:
     # or list the available templates.
     NOTEBOOKER_DISABLE_GIT: bool = False
 
+    # A boolean flag to dictate whether we should execute the notebook at the origin or not.
+    EXECUTE_AT_ORIGIN: bool = False
+
     # The serializer class we are using for storage, e.g. PyMongoResultSerializer
     SERIALIZER_CLS: DEFAULT_SERIALIZER = None
     # The dictionary of parameters which are used to initialize the serializer class above

notebooker/version.py

@@ -1 +1 @@
-__version__ = "0.6.2"
+__version__ = "0.6.3"

notebooker/web/static/package.json

@@ -1,6 +1,6 @@
 {
   "name": "notebooker",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "description": "Notebooker - Turn notebooks into reports",
   "dependencies": {
     "bootstrap-table": "1.20.2",

tests/regression/local_context/hello.txt

@@ -0,0 +1,2 @@
+Hello, World!
+This is a sample text file.

tests/regression/local_context/hello_module.py

@@ -0,0 +1,2 @@
+def print_hello():
+    print("Hello, World!")

tests/regression/local_context/local_import.ipynb

@@ -0,0 +1,73 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "c63308ae-fa0a-4cd3-8e48-c5c2f16836c2",
+   "metadata": {
+    "editable": true,
+    "execution": {
+     "iopub.execute_input": "2024-06-26T09:33:30.963587Z",
+     "iopub.status.busy": "2024-06-26T09:33:30.963239Z",
+     "iopub.status.idle": "2024-06-26T09:33:30.974539Z",
+     "shell.execute_reply": "2024-06-26T09:33:30.973885Z",
+     "shell.execute_reply.started": "2024-06-26T09:33:30.963543Z"
+    },
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Hello, World!\n"
+     ]
+    }
+   ],
+   "source": [
+    "import hello_module\n",
+    "\n",
+    "hello_module.print_hello()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "lines_to_next_cell": 0,
+    "tags": [
+     "parameters"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "plots = 5\n"
+   ],
+   "id": "e4425e0625d403a"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

tests/regression/local_context/local_read.py

@@ -0,0 +1,8 @@
+filename = "hello.txt"
+
+# Open the file and read its content
+with open(filename, "r") as file:
+    content = file.read()
+
+# Print the content
+print(content)

tests/regression/test_execute_with_local_context.py

@@ -0,0 +1,37 @@
+import datetime
+import os
+import uuid
+
+import pytest
+
+from notebooker.execute_notebook import _run_checks
+
+
+@pytest.fixture(scope="module")
+def py_template_base_dir():
+    import tests.regression.local_context as local_context
+
+    return os.path.abspath(local_context.__path__[0])
+
+
+def all_templates():
+    return ["local_read", "local_import"]
+
+
+@pytest.mark.parametrize("template_name", all_templates())
+def test_execution_of_templates_with_local_context(
+    template_name, template_dir, output_dir, flask_app, py_template_base_dir
+):
+    with flask_app.app_context():
+        _run_checks(
+            "job_id_{}".format(str(uuid.uuid4())[:6]),
+            datetime.datetime.now(),
+            template_name,
+            template_name,
+            output_dir,
+            template_dir,
+            {},
+            py_template_base_dir=py_template_base_dir,
+            execute_at_origin=True,
+            generate_pdf_output=False,
+        )