Merge pull request #167 from neutrons/livedata_systemtest

Add systemtest for livedata/postprocessing interaction

.env

@@ -4,7 +4,6 @@ DATABASE_USER=workflow
 DATABASE_PASS=workflow
 DATABASE_PORT=5432
 DATABASE_HOST=db
-SETTINGS_DIR=/var/www/workflow/app
 TIME_ZONE=America/New_York
 APP_SECRET='-0zoc$fl2fa&rmzeo#uh-qz-k+4^1)_9p1qwby1djzybqtl_nn'
 
@@ -17,3 +16,5 @@ WORKER_PASS=worker
 
 AMQ_BROKER=[["activemq", 61613]]
 AMQ_QUEUE=["/topic/SNS.COMMON.STATUS.WORKFLOW.0", "/topic/SNS.COMMON.STATUS.AUTOREDUCE.0", "/topic/SNS.*.APP.DASMON", "/topic/SNS.*.STATUS.DASMON", "/topic/SNS.*.SIGNAL.DASMON", "/topic/SNS.*.APP.SMS", "/topic/SNS.*.STATUS.SMS", "/topic/SNS.*.STATUS.POSTPROCESS", "/topic/SNS.COMMON.STATUS.ACK", "/topic/SNS.*.STATUS.PVSD", "/topic/HFIR.*.APP.DASMON", "/topic/HFIR.*.STATUS.DASMON", "/topic/HFIR.*.SIGNAL.DASMON", "/topic/HFIR.*.APP.SMS", "/topic/HFIR.*.STATUS.SMS", "/topic/HFIR.*.STATUS.POSTPROCESS", "/topic/HFIR.COMMON.STATUS.ACK", "/topic/HFIR.*.STATUS.PVSD"]
+
+LIVE_DATA_SERVER_DOMAIN=172.16.238.222

.github/workflows/ci.yml

@@ -13,8 +13,8 @@ jobs:
       run:
         shell: bash -l {0}
     steps:
-    - uses: actions/checkout@v2
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: actions/checkout@v4
+    - uses: conda-incubator/setup-miniconda@v3
       with:
         auto-update-conda: true
     - name: Setup libmamba solver

.github/workflows/publish.yml

@@ -3,6 +3,7 @@ name: Wheel + Docker publish
 on:
   push:
     branches:
+      - next
       - qa
       - main
 
@@ -13,17 +14,17 @@ env:
 jobs:
   build:
 
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash -l {0}
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
       - name: Setup libmamba solver
@@ -40,29 +41,28 @@ jobs:
         run: |
           conda activate webmon
           make wheel/all
-      - name: Get version
-        id: version
-        run: |
-          conda activate webmon
-          echo "::set-output name=version::$(versioningit src/webmon_app)"
 
       - name: Create tag version
         id: tag
         run: |
           conda activate webmon
-          echo "::set-output name=tag::$(versioningit src/webmon_app).b$(date +'%Y%m%d%H%M')"
+          echo "tag=$(versioningit src/webmon_app)" >> $GITHUB_OUTPUT
 
       - name: Create latest tag version
         id: latest_tag
         run: |
           case ${{ github.ref }} in
 
+          refs/heads/next)
+            echo "latest_tag=latest-dev" >> $GITHUB_OUTPUT
+            ;;
+
           refs/heads/qa)
-            echo "::set-output name=latest_tag::latest-test"
+            echo "latest_tag=latest-test" >> $GITHUB_OUTPUT
             ;;
 
           refs/heads/main)
-            echo "::set-output name=latest_tag::latest-prod"
+            echo "latest_tag=latest-prod" >> $GITHUB_OUTPUT
             ;;
 
           *)
@@ -72,10 +72,10 @@ jobs:
           esac
 
       - name: Upload wheels
-        uses: softprops/action-gh-release@v1
-        if: ${{ github.ref != 'refs/heads/next' }}
+        uses: softprops/action-gh-release@v2
+        if: ${{ github.ref == 'refs/heads/main' }}
         with:
-          tag_name: ${{ steps.version.outputs.version }}
+          tag_name: ${{ steps.tag.outputs.tag }}
           files: src/*/dist/*.whl
 
       - name: Check tag names
@@ -87,78 +87,122 @@ jobs:
         run: make SNSdata.tar.gz
 
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push amq_test_gen
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: Dockerfile.amq_test_gen
           tags: |
             ${{ env.REGISTRY }}/${{ github.repository }}/amq_test_gen:${{ steps.latest_tag.outputs.latest_tag }}
+          push: true
+
+      - name: Push amq_test_gen with version tag only for main branch
+        if: github.ref == 'refs/heads/main'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile.amq_test_gen
+          tags: |
             ${{ env.REGISTRY }}/${{ github.repository }}/amq_test_gen:${{ steps.tag.outputs.tag }}
           push: true
 
       - name: Build and push Autoreducer
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: Dockerfile.autoreducer
           tags: |
             ${{ env.REGISTRY }}/${{ github.repository }}/autoreducer:${{ steps.latest_tag.outputs.latest_tag }}
-            ${{ env.REGISTRY }}/${{ github.repository }}/autoreducer:${{ steps.tag.outputs.tag }}
           push: true
 
-      - name: Build and push Catalog
-        uses: docker/build-push-action@v2
+      - name: Push Autoreducer with version tag only for main branch
+        if: github.ref == 'refs/heads/main'
+        uses: docker/build-push-action@v6
         with:
           context: .
-          file: Dockerfile.catalog
+          file: Dockerfile.autoreducer
           tags: |
-            ${{ env.REGISTRY }}/${{ github.repository }}/catalog:${{ steps.latest_tag.outputs.latest_tag }}
-            ${{ env.REGISTRY }}/${{ github.repository }}/catalog:${{ steps.tag.outputs.tag }}
+            ${{ env.REGISTRY }}/${{ github.repository }}/autoreducer:${{ steps.tag.outputs.tag }}
           push: true
 
       - name: Build and push Dasmon
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: Dockerfile.dasmon
           tags: |
             ${{ env.REGISTRY }}/${{ github.repository }}/dasmon:${{ steps.latest_tag.outputs.latest_tag }}
+          push: true
+
+      - name: Push Dasmon with version tag only for main branch
+        if: github.ref == 'refs/heads/main'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile.dasmon
+          tags: |
             ${{ env.REGISTRY }}/${{ github.repository }}/dasmon:${{ steps.tag.outputs.tag }}
           push: true
 
       - name: Build and push pv_test_gen
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: Dockerfile.pv_test_gen
           tags: |
             ${{ env.REGISTRY }}/${{ github.repository }}/pv_test_gen:${{ steps.latest_tag.outputs.latest_tag }}
+          push: true
+
+      - name: Push pv_test_gen with version tag only for main branch
+        if: github.ref == 'refs/heads/main'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile.pv_test_gen
+          tags: |
             ${{ env.REGISTRY }}/${{ github.repository }}/pv_test_gen:${{ steps.tag.outputs.tag }}
           push: true
 
       - name: Build and push Webmon
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: Dockerfile.webmon
           tags: |
             ${{ env.REGISTRY }}/${{ github.repository }}/webmon:${{ steps.latest_tag.outputs.latest_tag }}
+          push: true
+
+      - name: Push Webmon with version tag only for main branch
+        if: github.ref == 'refs/heads/main'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile.webmon
+          tags: |
             ${{ env.REGISTRY }}/${{ github.repository }}/webmon:${{ steps.tag.outputs.tag }}
           push: true
 
       - name: Build and push Workflow
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: Dockerfile.workflow
           tags: |
             ${{ env.REGISTRY }}/${{ github.repository }}/workflow:${{ steps.latest_tag.outputs.latest_tag }}
+          push: true
+
+      - name: Push Workflow with version tag only for main branch
+        if: github.ref == 'refs/heads/main'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile.workflow
+          tags: |
             ${{ env.REGISTRY }}/${{ github.repository }}/workflow:${{ steps.tag.outputs.tag }}
           push: true

.github/workflows/systemtests.yml

@@ -13,8 +13,8 @@ jobs:
       run:
         shell: bash -l {0}
     steps:
-    - uses: actions/checkout@v2
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: actions/checkout@v4
+    - uses: conda-incubator/setup-miniconda@v3
       with:
         auto-update-conda: true
     - name: Setup libmamba solver
@@ -27,12 +27,10 @@ jobs:
         conda env create --file conda_environment.yml --quiet
         conda activate webmon
         conda env update --file conda_development.yml --quiet
-    - name: Build wheels
+    - name: Build requirements
       run: |
         conda activate webmon
-        make wheel/dasmon wheel/webmon wheel/workflow
-    - name: Build test data for autoreducer
-      run: make SNSdata.tar.gz
+        make all
     - name: Stand up docker containers
       run: docker-compose up --build -d
       env:

Makefile

@@ -17,7 +17,7 @@ help:
     # this nifty perl one-liner collects all comments headed by the double "#" symbols next to each target and recycles them as comments
 	@perl -nle'print $& if m{^[/a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}'
 
-all: wheel/dasmon wheel/webmon wheel/workflow SNSdata.tar.gz
+all: wheel/dasmon wheel/webmon wheel/workflow SNSdata.tar.gz ssl
 
 create/conda:  ## create conda environment "webmon" with file conda_environment.yml
 	conda env create --name webmon --file conda_environment.yml
@@ -121,6 +121,11 @@ SNSdata.tar.gz:  ## install SNS data for testing and limited info display
 	# it needs to be removed when the directory changes
 	tar czf SNSdata.tar.gz -C tests/data/ .
 
+ssl: nginx/nginx.crt nginx/nginx.key ## self-signed ssl certificates for livedata server
+
+nginx/nginx.crt nginx/nginx.key:
+	openssl req -x509 -out nginx/nginx.crt -keyout nginx/nginx.key -newkey rsa:2048 -nodes -sha256 --config nginx/san.cnf
+
 localdev/up:  ## create images and start containers for local development. Doesn't update python wheels, though.
 	docker-compose --file docker-compose.yml up --build
 

config/livedata_local_settings.py

@@ -0,0 +1 @@
+ALLOWED_HOSTS = ["*"]