Implement get_document() API

.github/workflows/test.yml

@@ -152,10 +152,12 @@ jobs:
       - name: Base Setup
         uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
         with:
-          python_version: "3.11"
+          python_version: "3.8"
+          dependency_type: minimum
 
-      - name: Install minimum versions
-        uses: jupyterlab/maintainer-tools/.github/actions/install-minimums@v1
+      - name: Install the Python dependencies
+        run: |
+          pip install -e ".[test]"
 
       - name: Run the unit tests
         run: |

docs/source/configuration.md

@@ -10,7 +10,7 @@ you happen to delete it, there shouldn't be any serious consequence either.
 There are a number of settings that you can change:
 
 ```bash
-# To enable or disable RTC(Real-Time Collaboration) (default: False).
+# To enable or disable RTC (Real-Time Collaboration) (default: False).
 # If True, RTC will be disabled.
 jupyter lab --YDocExtension.disable_rtc=True
 
@@ -29,3 +29,21 @@ jupyter lab --YDocExtension.document_cleanup_delay=100
 # The YStore class to use for storing Y updates (default: JupyterSQLiteYStore).
 jupyter lab --YDocExtension.ystore_class=pycrdt_websocket.ystore.TempFileYStore
 ```
+
+There is an experimental feature that is currently only supported by the
+[Jupyverse](https://github.com/jupyter-server/jupyverse) server
+(not yet with [jupyter-server](https://github.com/jupyter-server/jupyter_server),
+see the [issue #900](https://github.com/jupyter-server/jupyter_server/issues/900)):
+server-side execution. With this, running notebook code cells is not done in the frontend through
+the low-level kernel protocol over WebSocket API, but through a high-level REST API. Communication
+with the kernel is then delegated to the server, and cell outputs are populated in the notebook
+shared document. The frontend gets these outputs changes and shows them live. What this means is
+that the notebook state can be recovered even if the frontend disconnects, because cell outputs are
+not populated frontend-side but server-side.
+
+This feature is disabled by default, and can be enabled like so:
+```bash
+pip install "jupyterlab>=4.2.0b0"
+pip install "jupyverse[jupyterlab, auth]>=0.4.2"
+jupyverse --set kernels.require_yjs=true --set jupyterlab.server_side_execution=true
+```

docs/source/developer/python_api.rst

@@ -4,6 +4,25 @@
 Python API
 ==========
 
+``jupyter_collaboration`` instantiates :any:`YDocExtension` and stores it under ``serverapp.settings`` dictionary, under the ``"jupyter_collaboration"`` key.
+This instance can be used in other extensions to access the public API methods.
+
+For example, to access a read-only view of the shared notebook model in your jupyter-server extension, you can use the :any:`get_document` method:
+
+.. code-block::
+
+   collaboration = serverapp.settings["jupyter_collaboration"]
+   document = collaboration.get_document(
+       path='Untitled.ipynb',
+       content_type="notebook",
+       file_format="json"
+   )
+   content = document.get()
+
+
+API Reference
+-------------
+
 .. automodule:: jupyter_collaboration.app
   :members:
   :inherited-members:

jupyter_collaboration/app.py

@@ -3,16 +3,26 @@
 from __future__ import annotations
 
 import asyncio
+from typing import Literal
 
 from jupyter_server.extension.application import ExtensionApp
+from jupyter_ydoc import ydocs as YDOCS
+from jupyter_ydoc.ybasedoc import YBaseDoc
+from pycrdt import Doc
 from pycrdt_websocket.ystore import BaseYStore
 from traitlets import Bool, Float, Type
 
 from .handlers import DocSessionHandler, YDocWebSocketHandler
 from .loaders import FileLoaderMapping
+from .rooms import DocumentRoom
 from .stores import SQLiteYStore
-from .utils import AWARENESS_EVENTS_SCHEMA_PATH, EVENTS_SCHEMA_PATH
-from .websocketserver import JupyterWebsocketServer
+from .utils import (
+    AWARENESS_EVENTS_SCHEMA_PATH,
+    EVENTS_SCHEMA_PATH,
+    encode_file_path,
+    room_id_from_encoded_path,
+)
+from .websocketserver import JupyterWebsocketServer, RoomNotFound
 
 
 class YDocExtension(ExtensionApp):
@@ -57,6 +67,14 @@ class YDocExtension(ExtensionApp):
         directory.""",
     )
 
+    server_side_execution = Bool(
+        False,
+        config=True,
+        help="""Whether to execute notebooks in the server using the REST API, not using the kernel
+        protocol over WebSocket. The frontend only interacts with the notebook through its shared
+        model.""",
+    )
+
     def initialize(self):
         super().initialize()
         self.serverapp.event_logger.register_event_schema(EVENTS_SCHEMA_PATH)
@@ -74,7 +92,11 @@ def initialize_settings(self):
 
     def initialize_handlers(self):
         self.serverapp.web_app.settings.setdefault(
-            "page_config_data", {"disableRTC": self.disable_rtc}
+            "page_config_data",
+            {
+                "disableRTC": self.disable_rtc,
+                "serverSideExecution": self.server_side_execution,
+            },
         )
 
         # Set configurable parameters to YStore class
@@ -112,6 +134,43 @@ def initialize_handlers(self):
             ]
         )
 
+    async def get_document(
+        self: YDocExtension,
+        *,
+        path: str,
+        content_type: Literal["notebook", "file"],
+        file_format: Literal["json", "text"],
+        copy: bool = True,
+    ) -> YBaseDoc | None:
+        """Get a view of the shared model for the matching document.
+
+        If `copy=True`, the returned shared model is a fork, meaning that any changes
+         made to it will not be propagated to the shared model used by the application.
+        """
+        file_id_manager = self.serverapp.web_app.settings["file_id_manager"]
+        file_id = file_id_manager.index(path)
+
+        encoded_path = encode_file_path(file_format, content_type, file_id)
+        room_id = room_id_from_encoded_path(encoded_path)
+
+        try:
+            room = await self.ywebsocket_server.get_room(room_id)
+        except RoomNotFound:
+            return None
+
+        if isinstance(room, DocumentRoom):
+            if copy:
+                update = room.ydoc.get_update()
+
+                fork_ydoc = Doc()
+                fork_ydoc.apply_update(update)
+
+                return YDOCS.get(content_type, YDOCS["file"])(fork_ydoc)
+            else:
+                return room._document
+
+        return None
+
     async def stop_extension(self):
         # Cancel tasks and clean up
         await asyncio.wait(

jupyter_collaboration/handlers.py

@@ -26,6 +26,7 @@
     LogLevel,
     MessageType,
     decode_file_path,
+    room_id_from_encoded_path,
 )
 from .websocketserver import JupyterWebsocketServer
 
@@ -74,7 +75,7 @@ async def prepare(self):
             await self._websocket_server.started.wait()
 
         # Get room
-        self._room_id: str = self.request.path.split("/")[-1]
+        self._room_id: str = room_id_from_encoded_path(self.request.path)
 
         async with self._room_lock(self._room_id):
             if self._websocket_server.room_exists(self._room_id):

jupyter_collaboration/utils.py

@@ -70,3 +70,8 @@ def encode_file_path(format: str, file_type: str, file_id: str) -> str:
             path (str): File path.
     """
     return f"{format}:{file_type}:{file_id}"
+
+
+def room_id_from_encoded_path(encoded_path: str) -> str:
+    """Transforms the encoded path into a stable room identifier."""
+    return encoded_path.split("/")[-1]

packages/collaboration-extension/package.json

@@ -56,20 +56,20 @@
     "@jupyter/collaboration": "^2.0.11",
     "@jupyter/docprovider": "^2.0.11",
     "@jupyter/ydoc": "^1.1.0-a0",
-    "@jupyterlab/application": "^4.0.5",
-    "@jupyterlab/apputils": "^4.0.5",
-    "@jupyterlab/codemirror": "^4.0.5",
+    "@jupyterlab/application": "^4.2.0-beta.0",
+    "@jupyterlab/apputils": "^4.2.0-beta.0",
+    "@jupyterlab/codemirror": "^4.2.0-beta.0",
     "@jupyterlab/coreutils": "^6.0.5",
-    "@jupyterlab/docregistry": "^4.0.5",
-    "@jupyterlab/filebrowser": "^4.0.5",
-    "@jupyterlab/fileeditor": "^4.0.5",
-    "@jupyterlab/logconsole": "^4.0.5",
-    "@jupyterlab/notebook": "^4.0.5",
+    "@jupyterlab/docregistry": "^4.2.0-beta.0",
+    "@jupyterlab/filebrowser": "^4.2.0-beta.0",
+    "@jupyterlab/fileeditor": "^4.2.0-beta.0",
+    "@jupyterlab/logconsole": "^4.2.0-beta.0",
+    "@jupyterlab/notebook": "^4.2.0-beta.0",
     "@jupyterlab/services": "^7.0.5",
-    "@jupyterlab/settingregistry": "^4.0.5",
-    "@jupyterlab/statedb": "^4.0.5",
-    "@jupyterlab/translation": "^4.0.5",
-    "@jupyterlab/ui-components": "^4.0.5",
+    "@jupyterlab/settingregistry": "^4.2.0-beta.0",
+    "@jupyterlab/statedb": "^4.2.0-beta.0",
+    "@jupyterlab/translation": "^4.2.0-beta.0",
+    "@jupyterlab/ui-components": "^4.2.0-beta.0",
     "@lumino/commands": "^2.1.0",
     "@lumino/widgets": "^2.1.0",
     "y-protocols": "^1.0.5",
@@ -97,7 +97,8 @@
     "schemaDir": "./schema",
     "outputDir": "../../jupyter_collaboration/labextension",
     "disabledExtensions": [
-      "@jupyterlab/filebrowser-extension:defaultFileBrowser"
+      "@jupyterlab/filebrowser-extension:defaultFileBrowser",
+      "@jupyterlab/notebook-extension:cell-executor"
     ],
     "sharedPackages": {
       "@codemirror/state": {

packages/collaboration-extension/src/collaboration.ts

@@ -14,9 +14,11 @@ import {
   EditorExtensionRegistry,
   IEditorExtensionRegistry
 } from '@jupyterlab/codemirror';
+import { type MarkdownCell } from '@jupyterlab/cells';
+import { INotebookCellExecutor, runCell } from '@jupyterlab/notebook';
 import { WebSocketAwarenessProvider } from '@jupyter/docprovider';
 import { SidePanel, usersIcon } from '@jupyterlab/ui-components';
-import { URLExt } from '@jupyterlab/coreutils';
+import { PageConfig, URLExt } from '@jupyterlab/coreutils';
 import { ServerConnection } from '@jupyterlab/services';
 import { IStateDB, StateDB } from '@jupyterlab/statedb';
 import { ITranslator, nullTranslator } from '@jupyterlab/translation';
@@ -189,3 +191,63 @@ export const userEditorCursors: JupyterFrontEndPlugin<void> = {
     });
   }
 };
+
+export const notebookCellExecutor: JupyterFrontEndPlugin<INotebookCellExecutor> =
+  {
+    id: '@jupyter/collaboration-extension:notebook-cell-executor',
+    description:
+      'Add notebook cell executor that uses REST API instead of kernel protocol over WebSocket.',
+    autoStart: true,
+    provides: INotebookCellExecutor,
+    activate: (app: JupyterFrontEnd): INotebookCellExecutor => {
+      if (PageConfig.getOption('serverSideExecution') === 'true') {
+        return Object.freeze({ runCell: runCellServerSide });
+      }
+      return Object.freeze({ runCell });
+    }
+  };
+
+async function runCellServerSide({
+  cell,
+  notebook,
+  notebookConfig,
+  onCellExecuted,
+  onCellExecutionScheduled,
+  sessionContext,
+  sessionDialogs,
+  translator
+}: INotebookCellExecutor.IRunCellOptions): Promise<boolean> {
+  switch (cell.model.type) {
+    case 'markdown':
+      (cell as MarkdownCell).rendered = true;
+      cell.inputHidden = false;
+      onCellExecuted({ cell, success: true });
+      break;
+    case 'code': {
+      const kernelId = sessionContext?.session?.kernel?.id;
+      const settings = ServerConnection.makeSettings();
+      const apiURL = URLExt.join(
+        settings.baseUrl,
+        `api/kernels/${kernelId}/execute`
+      );
+      const cellId = cell.model.sharedModel.getId();
+      const documentId = `json:notebook:${notebook.sharedModel.getState(
+        'file_id'
+      )}`;
+      const body = `{"cell_id":"${cellId}","document_id":"${documentId}"}`;
+      const init = {
+        method: 'POST',
+        body
+      };
+      try {
+        await ServerConnection.makeRequest(apiURL, init, settings);
+      } catch (error: any) {
+        throw new ServerConnection.NetworkError(error);
+      }
+      break;
+    }
+    default:
+      break;
+  }
+  return Promise.resolve(true);
+}

packages/collaboration-extension/src/index.ts

@@ -19,7 +19,8 @@ import {
   menuBarPlugin,
   rtcGlobalAwarenessPlugin,
   rtcPanelPlugin,
-  userEditorCursors
+  userEditorCursors,
+  notebookCellExecutor
 } from './collaboration';
 import { sharedLink } from './sharedlink';
 
@@ -37,7 +38,8 @@ const plugins: JupyterFrontEndPlugin<any>[] = [
   rtcGlobalAwarenessPlugin,
   rtcPanelPlugin,
   sharedLink,
-  userEditorCursors
+  userEditorCursors,
+  notebookCellExecutor
 ];
 
 export default plugins;

packages/docprovider/jest.config.js

@@ -7,7 +7,11 @@ const jestJupyterLab = require('@jupyterlab/testing/lib/jest-config');
 
 const esModules = [
   '@codemirror',
+  '@microsoft',
+  'exenv-es6',
   '@jupyter/ydoc',
+  '@jupyter/react-components',
+  '@jupyter/web-components',
   '@jupyterlab/',
   'lib0',
   'nanoid',

pyproject.toml

@@ -27,7 +27,7 @@ classifiers = [
     "Framework :: Jupyter :: JupyterLab :: Extensions :: Prebuilt",
 ]
 dependencies = [
-    "jupyter_server>=2.0.0,<3.0.0",
+    "jupyter_server>=2.4.0,<3.0.0",
     "jupyter_ydoc>=2.0.0,<3.0.0",
     "pycrdt-websocket>=0.12.5,<0.13.0",
     "jupyter_events>=0.10.0",
@@ -44,12 +44,12 @@ dev = [
 ]
 test = [
     "coverage",
-    "jupyter_server[test]>=2.0.0",
+    "jupyter_server[test]>=2.4.0",
     "jupyter_server_fileid[test]",
     "pytest>=7.0",
     "pytest-cov",
     "websockets",
-    "importlib_metadata >=3.6; python_version<'3.10'",
+    "importlib_metadata >=4.8.3; python_version<'3.10'",
 ]
 docs = [
     "jupyterlab>=4.0.0",