Restructure to build image and run integration test on it

.github/actions/create-dev-env/action.yml

@@ -0,0 +1,35 @@
+---
+name: Build environment
+description: Create build environment
+
+inputs:
+    architecture:
+        description: architecture to be run on
+        required: true
+        type: string
+
+runs:
+    using: composite
+    steps:
+        # actions/setup-python doesn't support Linux arm64 runners
+        # See: https://github.com/actions/setup-python/issues/108
+        # python3 is manually preinstalled in the arm64 VM self-hosted runner
+        - name: Set Up Python 🐍
+          if: ${{ inputs.architecture == 'amd64' }}
+          uses: actions/setup-python@v4
+          with:
+              python-version: 3.x
+
+        - name: Install Dev Dependencies 📦
+          if: ${{ inputs.architecture == 'amd64' }}
+          run: |
+              pip install --upgrade pip
+              pip install --upgrade -r docker/requirements-dev.txt
+          shell: bash
+
+        - name: Install Dev Dependencies 📦
+          if: ${{ inputs.architecture == 'arm64' }}
+          run: |
+              pip install --upgrade pip
+              pip install --upgrade -r docker/requirements-dev-arm64.txt
+          shell: bash

.github/actions/integration-tests/action.yml

@@ -0,0 +1,42 @@
+---
+name: Downstream tests
+description: Integration downstream tests the bulid image
+
+runs:
+    using: composite
+
+    steps:
+        - name: Set jupyter token env
+          run: echo "JUPYTER_TOKEN=$(openssl rand -hex 32)" >> $GITHUB_ENV
+          shell: bash
+
+        - name: Run pytest to test image is working
+          run: TAG=newly-baked pytest tests_integration/test_image.py
+          shell: bash
+
+        # The Firefox and its engine geckodrive need do be installed manually to run
+        - name: Install Firefox
+          uses: browser-actions/setup-firefox@latest
+          with:
+              firefox-version: '96.0'
+
+        - name: Install geckodriver
+          run: |
+              wget -c https://github.com/mozilla/geckodriver/releases/download/v0.30.0/geckodriver-v0.30.0-linux64.tar.gz
+              tar xf geckodriver-v0.30.0-linux64.tar.gz -C /usr/local/bin
+          shell: bash
+
+        - name: Run pytest for firefox
+          run: TAG=newly-baked pytest --driver Firefox tests_integration/test_app.py
+          shell: bash
+
+        - name: Run pytest for Chrome
+          run: TAG=newly-baked pytest --driver Chrome tests_integration/test_app.py
+          shell: bash
+
+        - name: Upload screenshots as artifacts
+          uses: actions/upload-artifact@v3
+          with:
+              name: Screenshots-CI
+              path: screenshots/
+              if-no-files-found: error

.github/workflows/di-docker.yml

@@ -0,0 +1,45 @@
+---
+# Run basic tests for this app on the latest aiidalab-docker image.
+name: smoke tests on notebooks
+
+on: [push, pull_request]
+
+
+# https://docs.github.com/en/actions/using-jobs/using-concurrency
+concurrency:
+    # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
+
+jobs:
+
+    build-test:
+
+        runs-on: ubuntu-latest
+        continue-on-error: true
+
+        steps:
+            - name: Checkout Repo ⚡️
+              uses: actions/checkout@v3
+            - name: Set Up Python 🐍
+              uses: actions/setup-python@v4
+              with:
+                  python-version: 3.x
+
+            - name: Install Dev Dependencies 📦
+              run: |
+                  pip install --upgrade pip
+                  pip install --upgrade -r docker/requirements-dev.txt
+
+            - name: Build image 🛠
+              working-directory: docker
+              run: docker buildx bake -f docker-bake.hcl --load
+              env:
+                  # Use buildx
+                  DOCKER_BUILDKIT: 1
+                  # Full logs for CI build
+                  BUILDKIT_PROGRESS: plain
+              shell: bash
+
+            - name: Run tests ✅
+              uses: ./.github/actions/integration-tests

.gitignore

@@ -109,3 +109,6 @@ venv.bak/
 
 .DS_Store
 .vscode
+
+# screenshots
+screenshots/

docker/Dockerfile

@@ -0,0 +1,18 @@
+# syntax=docker/dockerfile:1
+FROM aiidalab/full-stack:latest
+
+# Copy whole repo and pre-install the dependencies and app to the tmp folder.
+# In the before notebook scripts the app will be re-installed by moving it to the app folder.
+ENV PREINSTALL_APP_FOLDER ${HOME}/aiidalab-widgets-base
+COPY --chown=${NB_UID}:${NB_GID} --from=src . ${PREINSTALL_APP_FOLDER}
+
+USER ${NB_USER}
+
+RUN cd ${PREINSTALL_APP_FOLDER} && \
+     # Remove all untracked files and directories. For example the setup lock flag file.
+     git clean -fx && \
+     pip install . --no-cache-dir && \
+     fix-permissions "${CONDA_DIR}" && \
+     fix-permissions "/home/${NB_USER}"
+
+WORKDIR "/home/${NB_USER}"

docker/docker-bake.hcl

@@ -0,0 +1,16 @@
+# docker-bake.hcl for building QeApp images
+group "default" {
+  targets = ["awb"]
+}
+
+variable "ORGANIZATION" {
+  default = "aiidalab"
+}
+
+target "awb" {
+  tags = ["${ORGANIZATION}/aiidalab-widgets-base:newly-baked"]
+  context = "."
+  contexts = {
+    src = ".."
+  }
+}

docker/requirements-dev.txt

@@ -0,0 +1,11 @@
+docker
+requests
+pre-commit
+pytest
+pytest-docker
+
+# test dependencies
+pytest-selenium
+pytest-html<4.0
+selenium~=4.9.0
+webdriver-manager

tests_notebooks/conftest.py

@@ -41,12 +41,13 @@ def docker_compose(docker_services):
 
 @pytest.fixture(scope="session")
 def aiidalab_exec(docker_compose):
-    def execute(command, user=None, **kwargs):
-        workdir = "/home/jovyan/apps/aiidalab-widgets-base"
+    def execute(command, user=None, workdir=None, **kwargs):
+        opts = "-T"
         if user:
-            command = f"exec --workdir {workdir} -T --user={user} aiidalab {command}"
-        else:
-            command = f"exec --workdir {workdir} -T aiidalab {command}"
+            opts = f"{opts} --user={user}"
+        if workdir:
+            opts = f"{opts} --workdir={workdir}"
+        command = f"exec {opts} aiidalab {command}"
 
         return docker_compose.execute(command, **kwargs)
 
@@ -56,17 +57,10 @@ def execute(command, user=None, **kwargs):
 @pytest.fixture(scope="session", autouse=True)
 def notebook_service(docker_ip, docker_services, aiidalab_exec):
     """Ensure that HTTP service is up and responsive."""
-    # Directory ~/apps/aiidalab-widgets-base/ is mounted by docker,
-    # make it writeable for jovyan user, needed for `pip install`
-    aiidalab_exec("chmod -R a+rw /home/jovyan/apps/aiidalab-widgets-base", user="root")
-
-    # Install AWB with extra dependencies for SmilesWidget
-    aiidalab_exec("pip install -U .[smiles]")
-
     # `port_for` takes a container port and returns the corresponding host port
     port = docker_services.port_for("aiidalab", 8888)
     url = f"http://{docker_ip}:{port}"
-    token = os.environ["JUPYTER_TOKEN"]
+    token = os.environ.get("JUPYTER_TOKEN", "aiidalab")
     docker_services.wait_until_responsive(
         timeout=30.0, pause=0.1, check=lambda: is_responsive(url)
     )
@@ -78,7 +72,7 @@ def selenium_driver(selenium, notebook_service):
     def _selenium_driver(nb_path):
         url, token = notebook_service
         url_with_token = urljoin(
-            url, f"apps/apps/aiidalab-widgets-base/{nb_path}?token={token}"
+            url, f"apps/aiidalab-widgets-base/{nb_path}?token={token}"
         )
         selenium.get(f"{url_with_token}")
         # By default, let's allow selenium functions to retry for 10s
@@ -106,7 +100,8 @@ def final_screenshot(request, screenshot_dir, selenium):
     Screenshot name is generated from the test function name
     by stripping the 'test_' prefix
     """
-    screenshot_name = f"{request.function.__name__[5:]}.png"
+    browser_name = selenium.capabilities["browserName"]
+    screenshot_name = f"{request.function.__name__[5:]}-{browser_name}.png"
     screenshot_path = Path.joinpath(screenshot_dir, screenshot_name)
     yield
     selenium.get_screenshot_as_file(screenshot_path)

tests_notebooks/docker-compose.yml

@@ -4,15 +4,12 @@ version: '3.4'
 services:
 
     aiidalab:
-        image: aiidalab/full-stack:${TAG:-latest}
+        image: ${REGISTRY:-}${QE_IMAGE:-aiidalab/aiidalab-widgets-base}:${TAG:-newly-baked}
         environment:
-            RMQHOST: messaging
             TZ: Europe/Zurich
             DOCKER_STACKS_JUPYTER_CMD: notebook
             SETUP_DEFAULT_AIIDA_PROFILE: 'true'
             AIIDALAB_DEFAULT_APPS: ''
-            JUPYTER_TOKEN: ${JUPYTER_TOKEN}
-        volumes:
-            - ..:/home/jovyan/apps/aiidalab-widgets-base
+            JUPYTER_TOKEN: ${JUPYTER_TOKEN:-aiidalab}
         ports:
-            - 8998:8888
+            - 0.0.0.0:${AIIDALAB_PORT:-8998}:8888