add broker and ETS testing

Dockerfile

@@ -25,7 +25,7 @@ LABEL maintainer="[email protected]"
 
 ENV TZ="Etc/UTC" \
     DEBIAN_FRONTEND="noninteractive" \
-    DEBIAN_PACKAGES="bash git python3-pip python3-setuptools"
+    DEBIAN_PACKAGES="bash curl git python3-pip python3-setuptools vim"
 
 # copy the app
 COPY . /app

README.md

@@ -15,9 +15,10 @@ wis2-gdc is a Reference Implementation of a WIS2 Global Discovery Catalogue.
 - on discovery metadata notifications, run the WCMP2 ATS via [pywcmp](https://github.com/wmo-im/pywcmp)
   - ETS
   - KPIs
+- publish ETS and KPI reports to local broker under `gdc-reports`
 - publish to a WIS2 GDC (OGC API - Records) using one of the supported transaction backends:
   - [OGC API - Features - Part 4: Create, Replace, Update and Delete](https://docs.ogc.org/DRAFTS/20-002.html)
-  - Elasticsearch direct
+  - Elasticsearch direct (default)
 
 ## Installation
 
@@ -55,6 +56,12 @@ source local.env
 # setup pywis-pubsub - sync WIS2 notification schema
 pywis-pubsub schema sync
 
+# setup backend
+wis2-gdc setup
+
+# teardown backend
+wis2-gdc teardown
+
 # connect to Global Broker
 # discovery metadata notifications will automatically trigger wis2-gdc to validate and publish
 # WCMP2 to the GDC identified in wis2-gdc.env (WIS2_GDC_GB)

docker/docker-compose.override.yml

@@ -20,9 +20,12 @@
 ###############################################################################
 
 services:
-  wis2-gdc-api:
-    ports:
-      - 80:80
   elasticsearch:
     ports:
       - 9200:9200
+  mosquitto:
+    ports:
+      - 1883:1883
+  wis2-gdc-api:
+    ports:
+      - 80:80

docker/docker-compose.yml

@@ -50,6 +50,16 @@ services:
     networks:
       - wis2-gdc-net
 
+  mosquitto:
+    container_name: mosquitto
+    restart: always
+    build:
+      context: ./mosquitto
+    env_file:
+      - wis2-gdc.env
+    networks:
+      - wis2-gdc-net
+
   wis2-gdc-management:
     container_name: wis2-gdc-management
     build:

docker/entrypoint.sh

@@ -24,9 +24,12 @@
 
 echo "START /entrypoint.sh"
 
-echo "Caching WCMP2 schemas"
+echo "Caching WNM schema"
 pywis-pubsub schema sync
 
+echo "Caching WCMP schemas"
+pywcmp bundle sync
+
 echo "Setting up discovery metadata backend"
 wis2-gdc setup --yes
 

docker/mosquitto/Dockerfile

@@ -0,0 +1,28 @@
+###############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+###############################################################################
+
+FROM eclipse-mosquitto:latest
+
+COPY mosquitto.conf /mosquitto/config/mosquitto.conf
+COPY acl.conf /mosquitto/config/acl.conf
+COPY entrypoint.sh /docker-entrypoint.sh
+
+RUN chmod +x /docker-entrypoint.sh

docker/mosquitto/acl.conf

@@ -0,0 +1,6 @@
+user everyone
+topic read gdc-reports/#
+
+user _USERNAME
+topic readwrite #
+topic read $SYS/#

docker/mosquitto/entrypoint.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+USERNAME=`echo $WIS2_GDC_BROKER_URL |awk -F/ '{print $3}' | awk -F@ '{print $1}' | awk -F: '{print $1}'`
+PASSWORD=`echo $WIS2_GDC_BROKER_URL |awk -F/ '{print $3}' | awk -F@ '{print $1}' | awk -F: '{print $2}'`
+
+echo "Setting mosquitto authentication"
+
+echo "USERNAME: $USERNAME"
+echo "PASSWORD: $PASSWORD"
+
+if [ ! -e "/mosquitto/config/password.txt" ]; then
+    echo "Adding wis2-gc users to mosquitto password file"
+    mosquitto_passwd -b -c /mosquitto/config/password.txt $USERNAME $PASSWORD
+    mosquitto_passwd -b /mosquitto/config/password.txt everyone everyone
+    chmod 644 /mosquitto/config/password.txt
+else
+    echo "Mosquitto password file already exists. Skipping wis2box user addition."
+fi
+
+
+sed -i "s#_USERNAME#$USERNAME#" /mosquitto/config/acl.conf
+
+/usr/sbin/mosquitto -c /mosquitto/config/mosquitto.conf

docker/mosquitto/mosquitto.conf

@@ -0,0 +1,14 @@
+persistence true
+persistence_location /mosquitto/data/
+log_dest file /mosquitto/log/mosquitto.log
+log_dest stdout
+log_timestamp_format %Y-%m-%dT%H:%M:%S
+password_file /mosquitto/config/password.txt
+max_queued_messages 1000
+
+# ACLs
+acl_file /mosquitto/config/acl.conf
+
+## MQTT Listener
+listener 1883
+protocol mqtt

docker/wis2-gdc.env

@@ -1,5 +1,6 @@
 export WIS2_GDC_API_URL=http://localhost
 export WIS2_GDC_BACKEND_TYPE=Elasticsearch
 export WIS2_GDC_BACKEND_CONNECTION=http://elasticsearch:9200/wis2-discovery-metadata
-export WIS2_GDC_GB=mqtt://everyone:everyone@localhost:1883
+export WIS2_GDC_BROKER_URL=mqtt://wis2-gdc:wis2-gdc@mosquitto:1883
+export WIS2_GDC_GB=mqtts://everyone:[email protected]:8883
 export WIS2_GDC_GB_TOPIC=origin/a/wis2/+/+/metadata/#

wis2-gdc.env

@@ -1,5 +1,6 @@
 export WIS2_GDC_API_URL=http://localhost
 export WIS2_GDC_BACKEND_TYPE=Elasticsearch
 export WIS2_GDC_BACKEND_CONNECTION=http://localhost:9200/wis2-discovery-metadata
+export WIS2_GDC_BROKER_URL=mqtt://wis2-gdc:wis2-gdc@localhost:1883
 export WIS2_GDC_GB=mqtt://everyone:everyone@localhost:1883
 export WIS2_GDC_GB_TOPIC=origin/a/wis2/+/+/metadata/#

wis2_gdc/__init__.py

@@ -21,7 +21,7 @@
 
 import click
 
-from wis2_gdc.registrar import register, setup
+from wis2_gdc.registrar import register, setup, teardown
 from wis2_gdc.sync import sync
 
 __version__ = '0.1.dev0'
@@ -38,3 +38,4 @@ def cli():
 cli.add_command(sync)
 cli.add_command(register)
 cli.add_command(setup)
+cli.add_command(teardown)

wis2_gdc/env.py

@@ -24,3 +24,4 @@
 API_URL = os.environ.get('WIS2_GDC_API_URL')
 BACKEND_TYPE = os.environ.get('WIS2_GDC_BACKEND_TYPE')
 BACKEND_CONNECTION = os.environ.get('WIS2_GDC_BACKEND_CONNECTION')
+BROKER_URL = os.environ.get('WIS2_GDC_BROKER_URL')

wis2_gdc/registrar.py

@@ -25,33 +25,51 @@
 
 import click
 
+from pywcmp.wcmp2.ets import WMOCoreMetadataProfileTestSuite2
 from pywis_pubsub import cli_options
+from pywis_pubsub.mqtt import MQTTPubSubClient
 
 from wis2_gdc.backend import BACKENDS
-from wis2_gdc.env import BACKEND_TYPE, BACKEND_CONNECTION
+from wis2_gdc.env import BACKEND_TYPE, BACKEND_CONNECTION, BROKER_URL
 
 LOGGER = logging.getLogger(__name__)
 
 
 class Registrar:
     def __init__(self):
         self.metadata = None
+        self.broker = MQTTPubSubClient(BROKER_URL)
 
     def register(self, metadata: dict):
         self.metadata = metadata
         LOGGER.debug(f'Metadata: {self.metadata}')
-        LOGGER.debug(f'Publishing metadata to {BACKEND_TYPE} ({BACKEND_CONNECTION})')  # noqa
+
+        ets_results = self._run_ets()
+        if ets_results['ets-report']['summary']['FAILED'] > 0:
+            LOGGER.debug('ETS errors; metadata not published')
+
+        LOGGER.info(f'Publishing metadata to {BACKEND_TYPE} ({BACKEND_CONNECTION})')  # noqa
         self._publish()
 
-    def _run_ets(self):
-        pass
+    def _run_ets(self) -> dict:
+        LOGGER.info('Running ETS')
+        ts = WMOCoreMetadataProfileTestSuite2(self.metadata)
+        try:
+            results = ts.run_tests(fail_on_schema_validation=True)
+            LOGGER.info('Publishing ETS report to broker')
+            topic = f"gdc-reports/ets/{self.metadata['id']}"
+            self.broker.pub(topic, json.dumps(results))
+            return results
+        except Exception as err:
+            LOGGER.error(err)
 
     def _run_kpi(self):
+        LOGGER.info('Running KPI')
         pass
 
     def _publish(self):
         backend = BACKENDS[BACKEND_TYPE]({'connection': BACKEND_CONNECTION})
-
+        LOGGER.info('Saving metadata to backend')
         backend.save(self.metadata)
 
 
@@ -64,14 +82,31 @@ def setup(ctx, bypass, verbosity='NOTSET'):
     """Create GDC backend"""
 
     if not bypass:
-        if not click.confirm('Create GDC backends?  This will overwrite existing collections', abort=True):  # noqa
+        if not click.confirm('Create GDC backend?  This will overwrite existing collections', abort=True):  # noqa
             return
 
     backend = BACKENDS[BACKEND_TYPE]({'connection': BACKEND_CONNECTION})
     LOGGER.debug(f'Backend: {backend}')
     backend.setup()
 
 
+@click.command()
+@click.pass_context
+@click.option('--yes', '-y', 'bypass', is_flag=True, default=False,
+              help='Bypass permission prompts')
+@cli_options.OPTION_VERBOSITY
+def teardown(ctx, bypass, verbosity='NOTSET'):
+    """Delete GDC backend"""
+
+    if not bypass:
+        if not click.confirm('Delete GDC backend?  This will remove existing collections', abort=True):  # noqa
+            return
+
+    backend = BACKENDS[BACKEND_TYPE]({'connection': BACKEND_CONNECTION})
+    LOGGER.debug(f'Backend: {backend}')
+    backend.teardown()
+
+
 @click.command()
 @click.pass_context
 @click.argument('path')