Add stability tests

.eslintignore

@@ -12,6 +12,7 @@ node_modules
 **/schemas
 **/themes
 **/templates
+**/widget_lib
 coverage
 *.map.js
 *.bundle.js
@@ -23,4 +24,5 @@ coverage
 .history/
 .vscode/
 
-ui-tests/playwright-report
+ui-tests/playwright-report
+tests/widget_lib/**/extension.js

.github/workflows/ui-tests.yml

@@ -26,20 +26,40 @@ jobs:
           jupyter labextension develop . --overwrite
           cd ui-tests
           jlpm install --frozen-lockfile
-
-      - name: Launch Voila
-        run: |
-          cd ui-tests
-          # Mount a volume to overwrite the server configuration
-          jlpm start 2>&1 > /tmp/jupyterlab_server.log &
+          cd ../tests/widget_lib
+          pip install .
 
       - name: Install browser
         run: |
           cd ui-tests
           # Install only Chromium browser
           jlpm playwright install chromium
 
-      - name: Wait for JupyterLab
+      - name: Launch Voila stability test server
+        run: |
+          cd ui-tests
+          jlpm start-stability 2>&1 > /tmp/jupyterlab_server.log &          
+
+      - name: Wait for Voila
+        uses: ifaxity/wait-on-action@v1
+        with:
+          resource: http-get://localhost:8868/
+          timeout: 360000
+
+      - uses: iterative/setup-cml@v1
+      - name: Stability test
+        shell: bash
+        run: |
+          cd ui-tests
+          jlpm run test-stability
+
+      - name: Launch Voila
+        run: |
+          cd ui-tests
+          # Mount a volume to overwrite the server configuration
+          jlpm start 2>&1 > /tmp/jupyterlab_server.log &
+
+      - name: Wait for Voila
         uses: ifaxity/wait-on-action@v1
         with:
           resource: http-get://localhost:8866/
@@ -105,6 +125,16 @@ jobs:
           path: |
             ui-tests/playwright-report
 
+      - name: Update snapshots
+        if: failure()
+        run: |
+          cd ui-tests
+          # remove previous snapshots from other browser
+          jlpm rimraf "tests/**/*-snapshots/*.png"
+          jlpm rimraf "stability_test/**/*-snapshots/*.png"
+          # generate new snapshots
+          jlpm run test:update --browser ${{ matrix.browser }}
+
       - name: Print JupyterLab logs
         if: always()
         run: |

.gitignore

@@ -40,4 +40,8 @@ tsconfig.tsbuildinfo
 ui-tests/playwright-report
 ui-tests/test-results
 ui-tests/benchmark-results
-ui-tests/jlab_root
+ui-tests/jlab_root
+
+tests/widget_lib/**/docs
+tests/widget_lib/**/labextension
+tests/widget_lib/**/nbextension/index.*

docs/source/contribute.rst

@@ -217,6 +217,40 @@ Finally, to run the tests:
 
    python -m pytest
 
+UI tests with `Playwright`
+==========================
+
+Install the test dependencies and the mock widgets:
+
+.. code-block:: bash
+
+   python -m pip install jupyterlab~=3.0 numpy bqplot matplotlib ipympl ipyvolume scipy
+   cd tests/widget_lib
+   pip install .
+
+Install `Playwright` and `Chromium` browser:
+
+.. code-block:: bash
+
+   cd ui-tests
+   jlpm install --frozen-lockfile
+   jlpm playwright install chromium
+
+Start `Voilà` server and run the snapshot tests: 
+
+.. code-block:: bash
+
+   # In ui-tests folder
+   jlpm start
+   jlpm test
+
+To run the stability tests:
+
+.. code-block:: bash
+
+   jlpm start-stability
+   jlpm test-stability
+
 Editing templates
 =================
 

tests/widget_lib/MANIFEST.in

@@ -0,0 +1,25 @@
+include LICENSE.txt
+
+include pyproject.toml
+
+
+include tsconfig.json
+include package.json
+include webpack.config.js
+include widget_lib/labextension/*.tgz
+
+
+# Javascript files
+graft widget_lib/nbextension
+graft src
+graft css
+prune **/node_modules
+prune coverage
+prune lib
+
+# Patterns to exclude from any directory
+global-exclude *~
+global-exclude *.pyc
+global-exclude *.pyo
+global-exclude .git
+global-exclude .ipynb_checkpoints

tests/widget_lib/README.md

@@ -0,0 +1 @@
+## Collection of bugged widgets used to test the stability of `Voila`.

tests/widget_lib/babel.config.js

@@ -0,0 +1,13 @@
+module.exports = {
+  sourceMap: 'inline',
+  presets: [
+    [
+      '@babel/preset-env',
+      {
+        targets: {
+          node: 'current'
+        }
+      }
+    ]
+  ]
+};

tests/widget_lib/install.json

@@ -0,0 +1,5 @@
+{
+  "packageManager": "python",
+  "packageName": "widget_lib",
+  "uninstallInstructions": "Use your Python package manager (pip, conda, etc.) to uninstall the package widget_lib"
+}

tests/widget_lib/package.json

@@ -0,0 +1,94 @@
+{
+  "name": "widget_lib",
+  "version": "0.1.0",
+  "description": "A custom Jupyter Widget library for testing Voila",
+  "keywords": [
+    "jupyter",
+    "jupyterlab",
+    "jupyterlab-extension",
+    "widgets"
+  ],
+  "files": [
+    "lib/**/*.js",
+    "dist/*.js",
+    "css/*.css"
+  ],
+  "homepage": "https://github.com//widget_lib",
+  "bugs": {
+    "url": "https://github.com//widget_lib/issues"
+  },
+  "license": "BSD-3-Clause",
+  "author": {
+    "name": "Trung Le",
+    "email": ""
+  },
+  "main": "lib/index.js",
+  "types": "./lib/index.d.ts",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com//widget_lib"
+  },
+  "scripts": {
+    "build": "yarn run build:lib && yarn run build:nbextension && yarn run build:labextension:dev",
+    "build:prod": "yarn run build:lib && yarn run build:nbextension && yarn run build:labextension",
+    "build:labextension": "jupyter labextension build .",
+    "build:labextension:dev": "jupyter labextension build --development True .",
+    "build:lib": "tsc",
+    "build:nbextension": "webpack",
+    "clean": "yarn run clean:lib && yarn run clean:nbextension && yarn run clean:labextension",
+    "clean:lib": "rimraf lib",
+    "clean:labextension": "rimraf widget_lib/labextension",
+    "clean:nbextension": "rimraf widget_lib/nbextension/static/index.js",
+    "lint": "eslint . --ext .ts,.tsx --fix",
+    "lint:check": "eslint . --ext .ts,.tsx",
+    "prepack": "yarn run build:lib",
+    "test": "jest",
+    "watch": "npm-run-all -p watch:*",
+    "watch:lib": "tsc -w",
+    "watch:nbextension": "webpack --watch --mode=development",
+    "watch:labextension": "jupyter labextension watch ."
+  },
+  "dependencies": {
+    "@jupyter-widgets/base": "^1.1.10 || ^2.0.0 || ^3.0.0 || ^4.0.0"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.5.0",
+    "@babel/preset-env": "^7.5.0",
+    "@jupyterlab/builder": "^3.0.0",
+    "@phosphor/application": "^1.6.0",
+    "@phosphor/widgets": "^1.6.0",
+    "@types/jest": "^26.0.0",
+    "@types/webpack-env": "^1.13.6",
+    "@typescript-eslint/eslint-plugin": "^3.6.0",
+    "@typescript-eslint/parser": "^3.6.0",
+    "acorn": "^7.2.0",
+    "css-loader": "^3.2.0",
+    "eslint": "^7.4.0",
+    "eslint-config-prettier": "^6.11.0",
+    "eslint-plugin-prettier": "^3.1.4",
+    "fs-extra": "^7.0.0",
+    "identity-obj-proxy": "^3.0.0",
+    "jest": "^26.0.0",
+    "mkdirp": "^0.5.1",
+    "npm-run-all": "^4.1.3",
+    "prettier": "^2.0.5",
+    "rimraf": "^2.6.2",
+    "source-map-loader": "^1.1.3",
+    "style-loader": "^1.0.0",
+    "ts-jest": "^26.0.0",
+    "ts-loader": "^8.0.0",
+    "typescript": "~4.1.3",
+    "webpack": "^5.0.0",
+    "webpack-cli": "^4.0.0"
+  },
+  "jupyterlab": {
+    "extension": "lib/plugin",
+    "outputDir": "widget_lib/labextension/",
+    "sharedPackages": {
+      "@jupyter-widgets/base": {
+        "bundled": false,
+        "singleton": true
+      }
+    }
+  }
+}

tests/widget_lib/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["jupyter_packaging==0.7.9", "jupyterlab==3.*", "setuptools>=40.8.0", "wheel"]
+build-backend = "setuptools.build_meta"

tests/widget_lib/setup.cfg

@@ -0,0 +1,7 @@
+[bdist_wheel]
+universal=1
+
+[metadata]
+long_description = file: README.md
+long_description_content_type = text/markdown
+license_file = LICENSE.txt