diff --git a/documentation/docs/assets/img/FTP-not-a-valid-path.png b/documentation/docs/assets/img/FTP-not-a-valid-path.png new file mode 100644 index 00000000..9c33127f Binary files /dev/null and b/documentation/docs/assets/img/FTP-not-a-valid-path.png differ diff --git a/documentation/docs/assets/img/csv2bufr-station-not-found.png b/documentation/docs/assets/img/csv2bufr-station-not-found.png new file mode 100644 index 00000000..5439ed89 Binary files /dev/null and b/documentation/docs/assets/img/csv2bufr-station-not-found.png differ diff --git a/documentation/docs/assets/img/grafana_success.png b/documentation/docs/assets/img/grafana_success.png index e4afe9ed..d9906b87 100644 Binary files a/documentation/docs/assets/img/grafana_success.png and b/documentation/docs/assets/img/grafana_success.png differ diff --git a/documentation/docs/assets/img/minio-admin-create-new-path.png b/documentation/docs/assets/img/minio-admin-create-new-path.png index 6d143725..48e04958 100644 Binary files a/documentation/docs/assets/img/minio-admin-create-new-path.png and b/documentation/docs/assets/img/minio-admin-create-new-path.png differ diff --git a/documentation/docs/assets/img/minio-admin-uploaded-file.png b/documentation/docs/assets/img/minio-admin-uploaded-file.png index d8badace..285178e5 100644 Binary files a/documentation/docs/assets/img/minio-admin-uploaded-file.png and b/documentation/docs/assets/img/minio-admin-uploaded-file.png differ diff --git a/documentation/docs/assets/img/minio-ui-create-new-path.png b/documentation/docs/assets/img/minio-ui-create-new-path.png new file mode 100644 index 00000000..1055a6be Binary files /dev/null and b/documentation/docs/assets/img/minio-ui-create-new-path.png differ diff --git a/documentation/docs/assets/img/minio-ui-create-path.png b/documentation/docs/assets/img/minio-ui-create-path.png new file mode 100644 index 00000000..0b86ed60 Binary files /dev/null and b/documentation/docs/assets/img/minio-ui-create-path.png differ diff --git a/documentation/docs/assets/img/minio-ui-upload-file.png b/documentation/docs/assets/img/minio-ui-upload-file.png new file mode 100644 index 00000000..02ba3fee Binary files /dev/null and b/documentation/docs/assets/img/minio-ui-upload-file.png differ diff --git a/documentation/docs/assets/img/minio-ui.png b/documentation/docs/assets/img/minio-ui.png index e5750fb4..5201ecf7 100644 Binary files a/documentation/docs/assets/img/minio-ui.png and b/documentation/docs/assets/img/minio-ui.png differ diff --git a/documentation/docs/assets/img/minio-wis2box-incoming-dataset-folder.png b/documentation/docs/assets/img/minio-wis2box-incoming-dataset-folder.png new file mode 100644 index 00000000..209d65ce Binary files /dev/null and b/documentation/docs/assets/img/minio-wis2box-incoming-dataset-folder.png differ diff --git a/documentation/docs/assets/img/mqtt-explorer-global-broker-msg-origin.png b/documentation/docs/assets/img/mqtt-explorer-global-broker-msg-origin.png new file mode 100644 index 00000000..00986f2d Binary files /dev/null and b/documentation/docs/assets/img/mqtt-explorer-global-broker-msg-origin.png differ diff --git a/documentation/docs/assets/img/mqtt-explorer-global-broker-msg-properties.png b/documentation/docs/assets/img/mqtt-explorer-global-broker-msg-properties.png new file mode 100644 index 00000000..4b6f1b3f Binary files /dev/null and b/documentation/docs/assets/img/mqtt-explorer-global-broker-msg-properties.png differ diff --git a/documentation/docs/assets/img/mqtt-explorer-global-broker-sub-origin-cache-synop.png b/documentation/docs/assets/img/mqtt-explorer-global-broker-sub-origin-cache-synop.png new file mode 100644 index 00000000..280b2a5e Binary files /dev/null and b/documentation/docs/assets/img/mqtt-explorer-global-broker-sub-origin-cache-synop.png differ diff --git a/documentation/docs/assets/img/mqtt-explorer-global-broker-sub-origin.png b/documentation/docs/assets/img/mqtt-explorer-global-broker-sub-origin.png new file mode 100644 index 00000000..0ae2cd3f Binary files /dev/null and b/documentation/docs/assets/img/mqtt-explorer-global-broker-sub-origin.png differ diff --git a/documentation/docs/assets/img/mqtt-explorer-sys-topic.png b/documentation/docs/assets/img/mqtt-explorer-sys-topic.png new file mode 100644 index 00000000..2624a6c0 Binary files /dev/null and b/documentation/docs/assets/img/mqtt-explorer-sys-topic.png differ diff --git a/documentation/docs/assets/img/mqtt-explorer-topics.png b/documentation/docs/assets/img/mqtt-explorer-topics.png index 0e232e3c..8369b35c 100644 Binary files a/documentation/docs/assets/img/mqtt-explorer-topics.png and b/documentation/docs/assets/img/mqtt-explorer-topics.png differ diff --git a/documentation/docs/assets/img/mqtt-explorer-wis2-notification-metadata.png b/documentation/docs/assets/img/mqtt-explorer-wis2-notification-metadata.png new file mode 100644 index 00000000..b79bb3f0 Binary files /dev/null and b/documentation/docs/assets/img/mqtt-explorer-wis2-notification-metadata.png differ diff --git a/documentation/docs/assets/img/mqtt-explorer-wis2box-broker-everyone-everyone.png b/documentation/docs/assets/img/mqtt-explorer-wis2box-broker-everyone-everyone.png new file mode 100644 index 00000000..874b5672 Binary files /dev/null and b/documentation/docs/assets/img/mqtt-explorer-wis2box-broker-everyone-everyone.png differ diff --git a/documentation/docs/assets/img/mqtt-explorer-wis2box-storage.png b/documentation/docs/assets/img/mqtt-explorer-wis2box-storage.png new file mode 100644 index 00000000..08dd1312 Binary files /dev/null and b/documentation/docs/assets/img/mqtt-explorer-wis2box-storage.png differ diff --git a/documentation/docs/assets/img/popup-existing-dataset-id.png b/documentation/docs/assets/img/popup-existing-dataset-id.png new file mode 100644 index 00000000..67311978 Binary files /dev/null and b/documentation/docs/assets/img/popup-existing-dataset-id.png differ diff --git a/documentation/docs/assets/img/synop2bufr-ex2-success.png b/documentation/docs/assets/img/synop2bufr-ex2-success.png index f52186ba..de684d9a 100644 Binary files a/documentation/docs/assets/img/synop2bufr-ex2-success.png and b/documentation/docs/assets/img/synop2bufr-ex2-success.png differ diff --git a/documentation/docs/assets/img/synop2bufr-toggle.png b/documentation/docs/assets/img/synop2bufr-toggle.png index 17cb467c..5932987f 100644 Binary files a/documentation/docs/assets/img/synop2bufr-toggle.png and b/documentation/docs/assets/img/synop2bufr-toggle.png differ diff --git a/documentation/docs/assets/img/winbox-webapp-credentials.png b/documentation/docs/assets/img/winbox-webapp-credentials.png new file mode 100644 index 00000000..663ff0ca Binary files /dev/null and b/documentation/docs/assets/img/winbox-webapp-credentials.png differ diff --git a/documentation/docs/assets/img/winbox-webapp-options.png b/documentation/docs/assets/img/winbox-webapp-options.png new file mode 100644 index 00000000..3da5d428 Binary files /dev/null and b/documentation/docs/assets/img/winbox-webapp-options.png differ diff --git a/documentation/docs/assets/img/winscp-ftp-connection.png b/documentation/docs/assets/img/winscp-ftp-connection.png new file mode 100644 index 00000000..e8c70eff Binary files /dev/null and b/documentation/docs/assets/img/winscp-ftp-connection.png differ diff --git a/documentation/docs/assets/img/winscp-student-vm-exercise-materials.png b/documentation/docs/assets/img/winscp-student-vm-exercise-materials.png index a5b70973..d9b85cdf 100644 Binary files a/documentation/docs/assets/img/winscp-student-vm-exercise-materials.png and b/documentation/docs/assets/img/winscp-student-vm-exercise-materials.png differ diff --git a/documentation/docs/assets/img/wis2-notification-metadata-links.png b/documentation/docs/assets/img/wis2-notification-metadata-links.png new file mode 100644 index 00000000..62e84ef0 Binary files /dev/null and b/documentation/docs/assets/img/wis2-notification-metadata-links.png differ diff --git a/documentation/docs/assets/img/wis2box-api-collections.png b/documentation/docs/assets/img/wis2box-api-collections.png index 6abe4338..c4789b31 100644 Binary files a/documentation/docs/assets/img/wis2box-api-collections.png and b/documentation/docs/assets/img/wis2box-api-collections.png differ diff --git a/documentation/docs/assets/img/wis2box-api-discovery-metadata.png b/documentation/docs/assets/img/wis2box-api-discovery-metadata.png new file mode 100644 index 00000000..d8787afe Binary files /dev/null and b/documentation/docs/assets/img/wis2box-api-discovery-metadata.png differ diff --git a/documentation/docs/assets/img/wis2box-api.png b/documentation/docs/assets/img/wis2box-api.png index f40f2bab..85020fd4 100644 Binary files a/documentation/docs/assets/img/wis2box-api.png and b/documentation/docs/assets/img/wis2box-api.png differ diff --git a/documentation/docs/assets/img/wis2box-create-new-dataset-form-initial.png b/documentation/docs/assets/img/wis2box-create-new-dataset-form-initial.png new file mode 100644 index 00000000..7bd78633 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-create-new-dataset-form-initial.png differ diff --git a/documentation/docs/assets/img/wis2box-create-new-dataset.png b/documentation/docs/assets/img/wis2box-create-new-dataset.png new file mode 100644 index 00000000..39290407 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-create-new-dataset.png differ diff --git a/documentation/docs/assets/img/wis2box-data-mappings.png b/documentation/docs/assets/img/wis2box-data-mappings.png new file mode 100644 index 00000000..68aca03c Binary files /dev/null and b/documentation/docs/assets/img/wis2box-data-mappings.png differ diff --git a/documentation/docs/assets/img/wis2box-dataset-editor-new-dataset.png b/documentation/docs/assets/img/wis2box-dataset-editor-new-dataset.png new file mode 100644 index 00000000..b2052856 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-dataset-editor-new-dataset.png differ diff --git a/documentation/docs/assets/img/wis2box-grafana-ui.png b/documentation/docs/assets/img/wis2box-grafana-ui.png index 28e9e3d4..e9ff1f58 100644 Binary files a/documentation/docs/assets/img/wis2box-grafana-ui.png and b/documentation/docs/assets/img/wis2box-grafana-ui.png differ diff --git a/documentation/docs/assets/img/wis2box-metadata-editor-part1.png b/documentation/docs/assets/img/wis2box-metadata-editor-part1.png new file mode 100644 index 00000000..e2255a78 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-metadata-editor-part1.png differ diff --git a/documentation/docs/assets/img/wis2box-metadata-editor-part2.png b/documentation/docs/assets/img/wis2box-metadata-editor-part2.png new file mode 100644 index 00000000..22576368 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-metadata-editor-part2.png differ diff --git a/documentation/docs/assets/img/wis2box-metadata-editor-part3.png b/documentation/docs/assets/img/wis2box-metadata-editor-part3.png new file mode 100644 index 00000000..ea82e668 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-metadata-editor-part3.png differ diff --git a/documentation/docs/assets/img/wis2box-metadata-validation-error.png b/documentation/docs/assets/img/wis2box-metadata-validation-error.png new file mode 100644 index 00000000..a338160a Binary files /dev/null and b/documentation/docs/assets/img/wis2box-metadata-validation-error.png differ diff --git a/documentation/docs/assets/img/wis2box-metadata-validation-success.png b/documentation/docs/assets/img/wis2box-metadata-validation-success.png new file mode 100644 index 00000000..aff6f91e Binary files /dev/null and b/documentation/docs/assets/img/wis2box-metadata-validation-success.png differ diff --git a/documentation/docs/assets/img/wis2box-minio-buckets.png b/documentation/docs/assets/img/wis2box-minio-buckets.png index e042e8b2..9f5f9614 100644 Binary files a/documentation/docs/assets/img/wis2box-minio-buckets.png and b/documentation/docs/assets/img/wis2box-minio-buckets.png differ diff --git a/documentation/docs/assets/img/wis2box-submit-dataset-success.png b/documentation/docs/assets/img/wis2box-submit-dataset-success.png new file mode 100644 index 00000000..d749c567 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-submit-dataset-success.png differ diff --git a/documentation/docs/assets/img/wis2box-ui-empty.png b/documentation/docs/assets/img/wis2box-ui-empty.png index 1e05f0b9..33bc0d89 100644 Binary files a/documentation/docs/assets/img/wis2box-ui-empty.png and b/documentation/docs/assets/img/wis2box-ui-empty.png differ diff --git a/documentation/docs/assets/img/wis2box-ui-new-dataset.png b/documentation/docs/assets/img/wis2box-ui-new-dataset.png new file mode 100644 index 00000000..cbd86506 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-ui-new-dataset.png differ diff --git a/documentation/docs/assets/img/wis2box-webapp-menu.png b/documentation/docs/assets/img/wis2box-webapp-menu.png new file mode 100644 index 00000000..ddfd0398 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-webapp-menu.png differ diff --git a/documentation/docs/assets/img/wis2box-webapp-monitor.png b/documentation/docs/assets/img/wis2box-webapp-monitor.png new file mode 100644 index 00000000..5c1ca3b6 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-webapp-monitor.png differ diff --git a/documentation/docs/assets/img/wis2downloader-dashboard.png b/documentation/docs/assets/img/wis2downloader-dashboard.png new file mode 100644 index 00000000..2851d459 Binary files /dev/null and b/documentation/docs/assets/img/wis2downloader-dashboard.png differ diff --git a/documentation/docs/assets/tables/aws-minimal.csv b/documentation/docs/assets/tables/aws-minimal.csv index 3894f4e8..4af5446d 100644 --- a/documentation/docs/assets/tables/aws-minimal.csv +++ b/documentation/docs/assets/tables/aws-minimal.csv @@ -28,7 +28,7 @@ "snow_depth","meters","Decimal","Snow depth at time of observation to 2 decimal places" "precipitation_intensity","kg m-2 h-1","Decimal","Intensity of precipitation at time of observation to 5 decimal places" "anemometer_height","meters","Decimal","Height of the anemometer above local ground to 2 decimal place" -"time_period_of_wind","minutes","Integer","Time period over which the wind speed and direction have been averegd. 10 minutes in normal cases or the number of minutes since a significant change occuring in the preceeding 10 minutes." +"time_period_of_wind","minutes","Integer","Defines the time period over which the wind speed and direction have been averaged. Set to -10 to indicate a measurement period over the preceeding 10 minutes." "wind_direction","degrees","Integer","Wind direction (at anemometer height) averaged from the caterisan components over the indicated time period, 0 decimal places" "wind_speed","ms-1","Decimal","Wind speed (at anemometer height) averaged from the cartesian components over the indicated time period, 1 decimal place" "maximum_wind_gust_direction_10_minutes","degrees","Integer","Highest 3 second average over the preceeding 10 minutes, 0 decimal places" diff --git a/documentation/docs/cheatsheets/docker.md b/documentation/docs/cheatsheets/docker.md index 738a1e4f..cc4005ff 100644 --- a/documentation/docs/cheatsheets/docker.md +++ b/documentation/docs/cheatsheets/docker.md @@ -6,37 +6,26 @@ title: Docker cheatsheet ## Overview -Docker allows for creating virtual envronments in an isolated manner in support +Docker allows for creating virtual environments in an isolated manner in support of virtualization of computing resources. The basic concept behind Docker is containerization, where software can run as services, interacting with other software containers, for example. The typical Docker workflow involves creating and building **images**, which are then run as live **containers**. +Docker is used to run the suite of services that make up wis2box using pre-built images. + ### Image management * List available images ```bash -docker images -``` - -* Build an image from a Dockerfile: - -```bash -cat << EOF > Dockerfile -FROM ubuntu:latest - -RUN apt-get update -RUN apt-get install –y nginx - -CMD ["echo", "Hello from my first Docker setup!"] -EOF +docker image ls ``` -* Building the image: +* Update an image: ```bash -docker build -t my-image:local . +docker pull my-image:latest ``` * Removing an image: @@ -53,12 +42,6 @@ docker rmi my-image:local docker volume ls ``` -* Create a volume: - -```bash -docker volume create my-volume -``` - * Display detailed information on a volume: ```bash @@ -79,12 +62,6 @@ docker volume prune ### Container Management -* Create a container from an image, with an interactive terminal (`-it`) and a mounted volume (`v`): - -```bash -docker run -it -v ${pwd}:/app my-image:local -``` - * Display a list of currently running containers: ```bash @@ -97,12 +74,6 @@ docker ps docker ps -a ``` -* Start a container: - -```bash -docker start my-image:local # starts a new container -``` - * Enter the interactive terminal of a running container: diff --git a/documentation/docs/cheatsheets/linux.md b/documentation/docs/cheatsheets/linux.md index 99506e11..0d64ef03 100644 --- a/documentation/docs/cheatsheets/linux.md +++ b/documentation/docs/cheatsheets/linux.md @@ -79,6 +79,12 @@ touch foo.txt echo "hi there" > test-file.txt ``` +* View the contents of a file: + +```bash +cat test-file.txt +``` + * Copy a file: ```bash diff --git a/documentation/docs/cheatsheets/wis2box.md b/documentation/docs/cheatsheets/wis2box.md index 101fe8bf..38e58f2d 100644 --- a/documentation/docs/cheatsheets/wis2box.md +++ b/documentation/docs/cheatsheets/wis2box.md @@ -11,26 +11,6 @@ wis2box runs as a suite of Docker Compose commands. The ``wis2box-ctl.py`` comm ## wis2box command essentials -### Building - -* Build all of wis2box: - -```bash -python3 wis2box-ctl.py build -``` - -* Build a specific wis2box Docker image: - -```bash -python3 wis2box-ctl.py build wis2box-management -``` - -* Update wis2box: - -```bash -python3 wis2box-ctl.py update -``` - ### Starting and stopping * Start wis2box: @@ -62,33 +42,3 @@ python3 wis2box-ctl.py login ```bash python3 wis2box-ctl.py login wis2box-api ``` - -### Design time commands (metadata management and publishing) - -!!! note - - You must be logged into the **wis2box-management** container to run the below commands - -* Publish discovery metadata: - -```bash -wis2box metadata discovery publish /path/to/discovery-metadata-file.yml -``` - -* Publish station metadata: - -```bash -wis2box metadata station publish-collection -``` - -* Add a dataset of observation collections from discovery metadata: - -```bash -wis2box data add-collection /path/to/discovery-metadata-file.yml -``` - -* Ingest data into the **wis2box-incoming** bucket to trigger processing and publishing: - -```bash -wis2box data ingest --topic-hierarchy topic.hierarchy.path --path /path/to/directory/of/data/files -``` diff --git a/documentation/docs/cheatsheets/yaml.md b/documentation/docs/cheatsheets/yaml.md deleted file mode 100644 index 8f7935ad..00000000 --- a/documentation/docs/cheatsheets/yaml.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -title: YAML cheatsheet ---- - -# YAML cheatsheet - -## Overview - -Many wis2box configuration files are managed in [YAML (YAML Ain't Markup Language)](https://yaml.org) -format. YAML allows for flexible configurations using indentation to do grouping and nesting. - - -## Indentation - -YAML is characterized by indentatation. It is important to ensure consistet indentatation in a YAML -fileiont. - -It is recommended to use **4 spaces** for a default indentation level. - -It is also strongly recommended to **NOT** use tabs for indentation. - -## A simple YAML file: - -```yaml -my-configuration: # <- this is a section - # this is a comment - # you can add comments in this manner as you wish - section: # <- this is nested section - subsection: # <- this is another nested section - value1: 123 # an integer - value2: 1.23 # a float - value3: '123' # a number forced into a string - value4: my value # a string (does not need quoting) - value4: [1, 2, 3, 4] # an array/list - - # lists can contain any data type, including more lists or sections - my-list: # another list, same result as value4 above - - item 1 - - item 2 - - item 3 -``` diff --git a/documentation/docs/index.md b/documentation/docs/index.en.md similarity index 92% rename from documentation/docs/index.md rename to documentation/docs/index.en.md index 03e0a90e..9575876d 100644 --- a/documentation/docs/index.md +++ b/documentation/docs/index.en.md @@ -2,7 +2,7 @@ title: Home --- -WMO logo +WMO logo # WIS2 in a box training WIS2 in a box ([wis2box](https://docs.wis2box.wis.wmo.int)) is a Free and Open Source (FOSS) Reference Implementation of a WMO WIS2 Node. The project provides a plug and play toolset to ingest, process, and publish weather/climate/water data using standards-based approaches in alignment with the WIS2 principles. wis2box also provides access to all data in the WIS2 network. wis2box is designed to have a low barrier to entry for data providers, providing enabling infrastructure and services for data discovery, access, and visualization. @@ -87,8 +87,6 @@ Container names (running images) are denoted in **bold**. ## Training location and materials -This training is always provided live at [https://training.wis2box.wis.wmo.int](https://training.wis2box.wis.wmo.int). - The training contents, wiki and issue tracker are managed on GitHub at [https://github.com/wmo-im/wis2box-training](https://github.com/wmo-im/wis2box-training). ## Printing the material @@ -98,8 +96,7 @@ File > Print > Save as PDF. ## Exercise materials -Exercise materials can be downloaded from the [exercise-materials.zip](https://training.wis2box.wis.wmo.int/exercise-materials.zip) zipfile. - +Exercise materials can be downloaded from the [exercise-materials.zip](/exercise-materials.zip) zipfile. ## Support @@ -107,7 +104,7 @@ For issues/bugs/suggestions or improvements/contributions to this training, plea All wis2box bugs, enhancements and issues can be reported on [GitHub](https://github.com/wmo-im/wis2box/issues). -For additional support of questions, please contact wis@wmo.int. +For additional support of questions, please contact wis2-support@wmo.int. As always, wis2box core documentation can always be found at [https://docs.wis2box.wis.wmo.int](https://docs.wis2box.wis.wmo.int). diff --git a/documentation/docs/index.es.md b/documentation/docs/index.es.md new file mode 100644 index 00000000..285fb5ec --- /dev/null +++ b/documentation/docs/index.es.md @@ -0,0 +1,109 @@ +--- +title: Inicio +--- + +Logotipo de la OMM +# Entrenamiento de "WIS2-in-a-box" + +"WIS2-in-a-box" ([wis2box](https://docs.wis2box.wis.wmo.int)) es una Implementación de Referencia de Código Abierto (FOSS) y gratuita de un Nodo WMO WIS2. El proyecto proporciona un conjunto de herramientas plug and play para ingerir, procesar y publicar datos meteorológicos/climáticos/hidrológicos utilizando enfoques basados en estándares alineados con los principios de WIS2. wis2box también proporciona acceso a todos los datos en la red de WIS2. wis2box está diseñado para tener una barrera de entrada baja para los proveedores de datos, ofreciendo infraestructura y servicios habilitadores para la búsqueda, acceso y visualización de datos. + +Este entrenamiento proporciona explicaciones paso a paso de varios aspectos del proyecto wis2box, así como una serie de ejercicios para ayudarte a publicar y descargar datos de WIS2. El entrenamiento se proporciona en forma de presentaciones generales y también de ejercicios prácticos. + +Los participantes podrán trabajar con datos de prueba y metadatos, así como integrar sus propios datos y metadatos. + +Este entrenamiento cubre una amplia gama de temas (instalación/configuración, publicación/descarga de datos, etc.). + +## Objetivos y resultados de aprendizaje + +Los objetivos de este entrenamiento son familiarizarse con lo siguiente: + +- Conceptos y componentes centrales de la arquitectura de WIS2 +- Formatos de datos y metadatos utilizados en WIS2 para descubrimiento y acceso +- Arquitectura y entorno de wis2box +- Funciones principales de wis2box: + - Gestión de metadatos + - Ingesta de datos y transformación al formato BUFR + - Broker MQTT para la publicación de mensajes de WIS2 + - Punto final HTTP para descarga de datos + - Punto final de API para acceso programático a datos + +## Navegación + +La navegación a la izquierda proporciona una tabla de contenido para todo el entrenamiento. + +La navegación a la derecha proporciona una tabla de contenido para una página específica. + +## Requisitos previos + +### Conocimientos + +- Comandos básicos de Linux (consulta la [hoja de trucos](cheatsheets/linux.md)) +- Conocimientos básicos de redes y protocolos de Internet + +### Software + +Este entrenamiento requiere las siguientes herramientas: + +- Una instancia que ejecute el sistema operativo Ubuntu (proporcionado por los instructores de la OMM durante las sesiones de entrenamiento locales) consulta [Accediendo a tu VM de estudiante](practical-sessions/accessing-your-student-vm.md#introduction) +- Cliente SSH para acceder a tu instancia +- Explorador MQTT en tu máquina local +- Cliente SCP y FTP para copiar archivos desde tu máquina local + +## Convenciones + +!!! question + + Una sección marcada de esta manera te invita a responder una pregunta. + +También notarás secciones de consejos y notas dentro del texto: + +!!! tip + + Los consejos ofrecen ayuda sobre cómo lograr mejor las tareas. + +!!! note + + Las notas proporcionan información adicional sobre el tema cubierto por la sesión práctica, así como la mejor manera de lograr tareas. + +Los ejemplos se indican de la siguiente manera: + +Configuración +``` {.yaml linenums="1"} +mi-colección-definida-en-yaml: + type: collection + title: mi título definido como un atributo yaml llamado title + description: mi descripción como un atributo yaml llamado description +``` + +Fragmentos que deben escribirse en una terminal/consola se indican como: + +```bash +echo 'Hello world' +``` + +Los nombres de los contenedores (imágenes en ejecución) se indican en **negrita**. + +## Ubicación y materiales de entrenamiento + +Los contenidos del entrenamiento, la wiki y el rastreador de problemas son gestionados en GitHub en [https://github.com/wmo-im/wis2box-training](https://github.com/wmo-im/wis2box-training). + +## Imprimir el material + +Este entrenamiento se puede exportar a PDF. Para guardar o imprimir este material de entrenamiento, ve a la [página de impresión](print_page) y selecciona Archivo > Imprimir > Guardar como PDF. + +## Materiales de ejercicio + +Los materiales de ejercicio se pueden descargar desde el archivo [exercise-materials.zip](/exercise-materials.zip) zipfile. + +## Soporte + +Para problemas/errores/sugerencias o mejoras/contribuciones a este entrenamiento, utiliza el [rastreador de problemas de GitHub](https://github.com/wmo-im/wis2box-training/issues). + +Todos los errores, mejoras y problemas de wis2box se pueden reportar en [GitHub](https://github.com/wmo-im/wis2box/issues). + +Para soporte adicional o preguntas, por favor contacta a wis@wmo.int. + +Como siempre, la documentación principal de wis2box siempre se puede encontrar en [https://docs.wis2box.wis.wmo.int](https://docs.wis2box.wis.wmo.int). + +¡Las contribuciones siempre son alentadas y bienvenidas! + diff --git a/documentation/docs/practical-sessions/accessing-your-student-vm.md b/documentation/docs/practical-sessions/accessing-your-student-vm.en.md similarity index 88% rename from documentation/docs/practical-sessions/accessing-your-student-vm.md rename to documentation/docs/practical-sessions/accessing-your-student-vm.en.md index 3ae299bc..a9ff2828 100644 --- a/documentation/docs/practical-sessions/accessing-your-student-vm.md +++ b/documentation/docs/practical-sessions/accessing-your-student-vm.en.md @@ -39,8 +39,8 @@ Your student VM has the following software pre-installed: The release archive for wis2box used in this training can be downloaded as follows: ```bash - wget https://github.com/wmo-im/wis2box/releases/download/1.0b5/wis2box-setup-1.0b5.zip - unzip wis2box-setup-1.0b5.zip + wget https://github.com/wmo-im/wis2box/releases/download/1.0b7/wis2box-setup-1.0b8.zip + unzip wis2box-setup-1.0b8.zip ``` You can always find the latest 'wis2box-setup' archive at [https://github.com/wmo-im/wis2box/releases](https://github.com/wmo-im/wis2box/releases). @@ -56,8 +56,6 @@ Your student VM has the following software pre-installed: ```bash pip3 install minio - pip3 install pywiscat - pip3 install pywis-pubsub ``` If you are using the student VM provided during local WIS2 training sessions, the required software will already be installed. @@ -128,8 +126,12 @@ To test that your user can run docker hello-world, run the following command: docker run hello-world ``` -returns: +This should pull the hello-world image and run a container that prints a message. + +Check that you see the following in the output: + ```console +... Hello from Docker! This message shows that your installation appears to be working correctly. ... @@ -144,10 +146,14 @@ ls ~/ ``` returns: ```console -exercise-materials wis2box-1.0b5 +exercise-materials wis2box-1.0b8 ``` -You can use WinSCP to connect to your instance and inspect the contents of your home directory and download or upload files between your VM and your local PC. +If you have WinSCP installed on your local PC, you can use it to connect to your student VM and inspect the contents of your home directory and download or upload files between your VM and your local PC. + +WinSCP is not required for the training, but it can be useful if you want to edit files on your VM using a text editor on your local PC. + +Here is how you can connect to your student VM using WinSCP: Open WinSCP and click on the "New Site". You can create a new SCP connection to your VM as follows: diff --git a/documentation/docs/practical-sessions/adding-gts-headers-to-wis2-notifications.md b/documentation/docs/practical-sessions/adding-gts-headers-to-wis2-notifications.md new file mode 100644 index 00000000..87f9daf5 --- /dev/null +++ b/documentation/docs/practical-sessions/adding-gts-headers-to-wis2-notifications.md @@ -0,0 +1,106 @@ +--- +title: Adding GTS headers to WIS2 notifications +--- + +# Adding GTS headers to WIS2 notifications + +!!! abstract "Learning outcomes" + + By the end of this practical session, you will be able to: + + - configure a mapping between filename and GTS headers + - ingest data with a filename that matches the GTS headers + - view the GTS headers in the WIS2 notifications + +## Introduction + +WMO Members wishing to stop their data transmission on GTS during the transition phase to WIS2 will need to add GTS headers to their WIS2 notifications. These headers enable the WIS2 to GTS gateway to forward the data to the GTS network. + +This allows Members having migrated to using a WIS2 node for data publication to disable their MSS system and ensure that their data is still available to Members not yet migrated to WIS2. + +The GTS property in the WIS2 Notification Message needs to be added as an additional property to the WIS2 Notification Message. The GTS property is a JSON object that contains the GTS headers that are required for the data to be forwarded to the GTS network. + +```json +{ + "gts": { + "ttaaii": "FTAE31", + "cccc": "VTBB" + } +} +``` + +Within wis2box you can add this to WIS2 Notifications automatically by providing an additional file named `gts_headers_mapping.csv` that contains the required information to map the GTS headers to the incoming filenames. + +This file should be placed in the directory defined by `WIS2BOX_HOST_DATADIR` in your `wis2box.env` and should have the following columns: + +- `string_in_filepath`: a string that is part of the filename that will be used to match the GTS headers +- `TTAAii`: the TTAAii header to be added to the WIS2 notification +- `CCCC`: the CCCC header to be added to the WIS2 notification + +## Preparation + +Ensure you have SSH access to your student VM and that your wis2box instance is up and running. + +Make sure that you are connected to the MQTT broker of your wis2box instance using MQTT Explorer. You can use the public credentials `everyone/everyone` to connect to the broker. + +Make sure you have a web browser open with the Grafana dashboard for your instance by going to `http://:3000` + +## creating `gts_headers_mapping.csv` + +To add GTS headers to your WIS2 notifications, a CSV file is required that maps GTS headers to incoming filenames. + +The CSV file should be named (exactly) `gts_headers_mapping.csv` and should be placed in the directory defined by `WIS2BOX_HOST_DATADIR` in your `wis2box.env`. + +## Exercise 1: providing a `gts_headers_mapping.csv` file + +Copy the file `exercise-materials/gts-headers-exercises/gts_headers_mapping.csv` to your wis2box instance and place it in the directory defined by `WIS2BOX_HOST_DATADIR` in your `wis2box.env`. + + +```bash +cp ~/exercise-materials/gts-headers-exercises/gts_headers_mapping.csv ~/wis2box-data +``` + +Then restart wis2box to apply the changes: + +```bash +cd ~/wis2box-1.0b8 +python3 wis2box-ctl.py restart +``` + +## Exercise 2: Ingesting data with GTS headers + +Copy the file `exercise-materials/gts-headers-exercises/A_SMRO01YRBK171200_C_EDZW_20240717120502.txt` to the directory defined by `WIS2BOX_HOST_DATADIR` in your `wis2box.env`: + +```bash +cp ~/exercise-materials/gts-headers-exercises/A_SMRO01YRBK171200_C_EDZW_20240717120502.txt ~/wis2box-data +``` + +Then login to the **wis2box-management** container: + +```bash +cd ~/wis2box-1.0b8 +python3 wis2box-ctl.py login +``` + +From the wis2box command line we can ingest the sample data file `A_SMRO01YRBK171200_C_EDZW_20240717120502.txt` into a specific dataset as follows: + +```bash +wis2box data ingest -p /data/wis2box/A_SMRO01YRBK171200_C_EDZW_20240717120502.txt --metadata-id urn:wmo:md:not-my-centre:core.surface-based-observations.synop +``` + +Make sure to replace the `metadata-id` option with the correct identifier for your dataset. + +Check the Grafana dashboard to see if the data was ingested correctly. If you see any WARNINGS or ERRORS, try to fix them and repeat the exercise the `wis2box data ingest` command. + +## Exercise 3: Viewing the GTS headers in the WIS2 Notification + +Go to the MQTT Explorer and check for the WIS2 Notification Message for the data you just ingested. + +The WIS2 Notification Message should contain the GTS headers you provided in the `gts_headers_mapping.csv` file. + +## Conclusion + +!!! success "Congratulations!" + In this practical session, you learned how to: + - add GTS headers to your WIS2 notifications + - verify GTS headers are made available via your wis2box installation diff --git a/documentation/docs/practical-sessions/automating-data-ingestion.en.md b/documentation/docs/practical-sessions/automating-data-ingestion.en.md new file mode 100644 index 00000000..7a1a59d9 --- /dev/null +++ b/documentation/docs/practical-sessions/automating-data-ingestion.en.md @@ -0,0 +1,275 @@ +--- +title: Automating data ingestion +--- + +# Automating data ingestion + +!!! abstract "Learning outcomes" + + By the end of this practical session, you will be able to: + + - understand how the data plugins of your dataset determine the data ingest workflow + - ingest data into wis2box using a script using the MinIO Python client + - ingest data into wis2box using the wis2box-ftp service + +## Introduction + +The **wis2box-management** container listens to events from the MinIO storage service to trigger data ingestion based on the data-plugins configured for your dataset. This allows you to upload data into the MinIO bucket and trigger the wis2box workflow to publish data on the WIS2 broker. + +The data-plugins define the Python modules that are loaded by the **wis2box-management** container and determine how the data is transformed and published. + +In the previous exercise you should have created a dataset using the template `surface-based-observations/synop` which included the following data-plugins: + +data-mappings + +When a file is uploaded to MinIO, wis2box will match the file to a dataset when the filepath contains the dataset id (`metadata_id`) and it will determine the data plugins to use based on the file extension and file pattern defined in the dataset mappings. + +In the previous sessions, we triggered the data ingest workflow by using the wis2box command line functionality, which uploads data to the MinIO storage in the correct path. + +The same steps can be done programmatically by using any MinIO or S3 client software, allowing you to automate your data ingestion as part of your operational workflows. + +If you are unable to adapt your system to upload data to MinIO directly, you can also use the **wis2box-ftp** service to forward data to the MinIO storage service. + +## Preparation + +Login to you student VM using your SSH client (PuTTY or other). + +Make sure wis2box is up and running: + +```bash +cd ~/wis2box-1.0b8/ +python3 wis2box-ctl.py start +python3 wis2box-ctl.py status +``` + +Make sure MQTT Explorer is running and connected to your instance. If you are still connected from the previous session, clear any previous messages you may have received from the queue. +This can be done by either by disconnecting and reconnecting or by clicking the trash can icon for the given topic. + +Make sure you have a web browser open with the Grafana dashboard for your instance by going to `http://:3000` + +And make sure you have a second tab open with the MinIO user interface at `http://:9001`. Remember you need to login with the `WIS2BOX_STORAGE_USER` and `WIS2BOX_STORAGE_PASSWORD` defined in your `wis2box.env` file. + +## Exercise 1: setup a Python script to ingest data into MinIO + +In this exercise we will use the MinIO Python client to copy data into MinIO. + +MinIO provides a Python client which can be installed as follows: + +```bash +pip3 install minio +``` + +On your student VM the 'minio' package for Python will already be installed. + +Go to the directory `exercise-materials/data-ingest-exercises`; this directory contains a sample script `copy_file_to_incoming.py` that uses the MinIO Python client to copy a file into MinIO. + +Try to run the script to copy the sample data file `csv-aws-example.csv` into the `wis2box-incoming` bucket in MinIO" as follows: + +```bash +cd ~/exercise-materials/data-ingest-exercises +python3 copy_file_to_incoming.py csv-aws-example.csv +``` + +!!! note + + You will get an error as the script is not configured to access the MinIO endpoint on your wis2box yet. + +The script needs to know the correct endpoint for accessing MinIO on your wis2box. If wis2box is running on your host, the MinIO endpoint is available at `http://:9000`. The script also needs to be updated with your storage password and the path in the MinIO bucket to store the data. + +!!! question "Update the script and ingest the CSV data" + + Edit the script `copy_file_to_incoming.py` to address the errors, using one of the following methods: + - From the command line: use the `nano` or `vim` text editor to edit the script + - Using WinSCP: start a new connection using File Protocol `SCP` and the same credentials as your SSH client. Navigate to the directory `exercise-materials/data-ingest-exercises` and edit `copy_file_to_incoming.py` using the built-in text editor + + Ensure that you: + + - define the correct MinIO endpoint for your host + - provide the correct storage password for your MinIO instance + - provide the correct path in the MinIO bucket to store the data + + Re-run the script to ingest the sample data file `csv-aws-example.csv` into MinIO: + + ```bash + python3 copy_file_to_incoming.py csv-aws-example.csv + ``` + + And make sure the errors are resolved. + +You can verify that the data was uploaded correctly by checking the MinIO user interface and seeing if the sample data is available in the correct directory in the `wis2box-incoming` bucket. + +You can use the Grafana dashboard to check the status of the data ingest workflow. + +Finally you can use MQTT Explorer to check if notifications were published for the data you ingested. You should see that the CSV data was transformed into BUFR format and that a WIS2 data notification was published with a "canonical" url to enable downloading the BUFR data. + +## Exercise 2: Ingesting binary data + +Next, we try to ingest binary data in BUFR format using the MinIO Python client. + +wis2box can ingest binary data in BUFR format using the `wis2box.data.bufr4.ObservationDataBUFR` plugin included in wis2box. + +This plugin will split the BUFR file into individual BUFR messages and publish each message to the MQTT broker. If the station for the corresponding BUFR message is not defined in the wis2box station metadata, the message will not be published. + +Since you used the `surface-based-observations/synop` template in the previous session you data mappings include the plugin `FM-12 data converted to BUFR` for the dataset mappings. This plugin loads the module `wis2box.data.synop2bufr.ObservationDataSYNOP2BUFR` to ingest the data. + +!!! question "Ingesting binary data in BUFR format" + + Run the following command to copy the binary data file `bufr-example.bin` into the `wis2box-incoming` bucket in MinIO: + + ```bash + python3 copy_file_to_incoming.py bufr-example.bin + ``` + +Check the Grafana dashboard and MQTT Explorer to see if the test-data was successfully ingested and published and if you see any errors, try to resolve them. + +!!! question "Verify the data ingest" + + How many messages were published to the MQTT broker for this data sample? + +??? success "Click to reveal answer" + + If you successfully ingested and published the last data sample, you should have received 10 new notifications on the wis2box MQTT broker. Each notification correspond to data for one station for one observation timestamp. + + The plugin `wis2box.data.bufr4.ObservationDataBUFR` splits the BUFR file into individual BUFR messages and publishes one message for each station and observation timestamp. + +## Exercise 3: Ingesting SYNOP data in ASCII format + +In the previous session we used the SYNOP form in the **wis2box-webapp** to ingest SYNOP data in ASCII format. You can also ingest SYNOP data in ASCII format by uploading the data into MinIO. + +In the previous session you should have created a dataset which included the plugin 'FM-12 data converted to BUFR' for the dataset mappings: + +dataset-mappings + +This plugin loads the module `wis2box.data.synop2bufr.ObservationDataSYNOP2BUFR` to ingest the data. + +Try to use the MinIO Python client to ingest the test data `synop-202307.txt` and `synop-202308.txt` into your wis2box instance. + +Note that the 2 files contain the same content, but the filename is different. The filename is used to determine the date of the data sample. + +The synop2bufr plugin relies on a file-pattern to extract the date from the filename. The first group in the regular expression is used to extract the year and the second group is used to extract the month. + +!!! question "Ingest FM-12 SYNOP data in ASCII format" + + Go back to the MinIO interface in your browse and navigate to the `wis2box-incoming` bucket and into the path where you uploaded the test data in the previous exercise. + + Upload the new files in the correct path in the `wis2box-incoming` bucket in MinIO to trigger the data ingest workflow. + + Check the Grafana dashboard and MQTT Explorer to see if the test data was successfully ingested and published. + + What is the difference in the `properties.datetime` between the two messages published to the MQTT broker? + +??? success "Click to reveal answer" + + Check the properties of the last 2 notifications in MQTT Explorer and you will note that one notification has: + + ```{.copy} + "properties": { + "data_id": "wis2/urn:wmo:md:nl-knmi-test:surface-based-observations.synop/WIGOS_0-20000-0-60355_20230703T090000", + "datetime": "2023-07-03T09:00:00Z", + ... + ``` + + and the other notification has: + + ```{.copy} + "properties": { + "data_id": "wis2/urn:wmo:md:nl-knmi-test:surface-based-observations.synop/WIGOS_0-20000-0-60355_20230803T090000", + "datetime": "2023-08-03T09:00:00Z", + ... + ``` + + The filename was used to determine the year and month of the data sample. + +## Exercise 4: ingesting data using the wis2box-ftp service + +You can add an additional service that adds an ftp-endpoint on your wis2box-instance. This service will forward data uploaded via ftp to the MinIO storage service, preserving the directory structure of the uploaded data. + +To use the `docker-compose.wis2box-ftp.yml` template included in wis2box, you need to pass some additional environment variables to the wis2box-ftp service. + +You can use the file `wis2box-ftp.env` file from the `exercise-materials/` directory to define the required environment variables. Start by copying the file to the `wis2box-1.0b8` directory: + +```bash +cp ~/exercise-materials/data-ingest-exercises/wis2box-ftp.env ~/wis2box-1.0b8/ +``` + +!!! question "Configuring and starting the wis2box-ftp service" + + Edit the file `wis2box-ftp.env` to define the required environment variables: + + - `FTP_USER`: the username for the ftp-endpoint (to be defined by the user) + - `FTP_PASS`: the password for the ftp-endpoint (to be defined by the user) + - `FTP_HOST`: the hostname of your wis2box-instance (e.g. `username.wis2.training`) + - `WIS2BOX_STORAGE_USERNAME`: the MinIO storage user (e.g. `wis2box`) + - `WIS2BOX_STORAGE_PASSWORD`: the MinIO storage password (see your `wis2box.env` file) + - `WIS2BOX_STORAGE_ENDPOINT`: the MinIO storage endpoint, you can leave this set to `http://minio:9000` when running the wis2box-ftp on the same docker network as the MinIO service. + + You can use the `nano` or `vim` text editor to edit the file or the built-in text editor of WinSCP. + + Then start the wis2box-ftp service using the following command: + + ```bash + cd ~/wis2box-1.0b8/ + docker compose -f docker-compose.wis2box-ftp.yml -p wis2box_project --env-file wis2box-ftp.env up -d + ``` + + NOTE: the option `-p wis2box_project` is used to ensure the wis2box-ftp service is started in the same docker network as the MinIO service for wis2box. + + You can check if the wis2box-ftp service is running using the following command: + + ```bash + docker logs wis2box-ftp + ``` + +To test the wis2box-ftp service, you can use an ftp client to upload a file to the ftp-endpoint on your wis2box-instance. The credentials for the ftp-endpoint are the ones you defined in the `wis2box-ftp.env` file by the `FTP_USER` and `FTP_PASS` environment variables. + +Using WinSCP, your connection would look as follows: + +winscp-ftp-connection + +In WinSCP, right-click and select *New*->*Directory* to create a new directory on the FTP endpoint. + +Uploading *randomfile.txt* to the directory *not-a-valid-path*: + +FTP-not-a-valid-path + +will result in the following message on the wis2box Grafana dashboard: + +*ERROR - Path validation error: Could not match http://minio:9000/wis2box-incoming/not-a-valid-path/randomfile.txt to dataset, path should include one of the following: ...* + +The file was forwarded by the wis2box-ftp service to the 'wis2box-incoming' bucket in MinIO, but the path did not match any of the dataset identifiers defined in your wis2box instance, resulting in an error. + +You can also use `ftp` from the command line: + +```bash +ftp username.wis2.training +``` +Login using the credentials defined in `wis2box-ftp.env` for the `FTP_USER` and `FTP_PASS` environment variables, and then create a directory and upload a file as follows: + +```bash +mkdir not-a-valid-path +cd not-a-valid-path +put ~/exercise-materials/data-ingest-exercises/synop.txt synop.txt +``` + +This will result a "Path validation error" in the Grafana dashboard indicating that the file was uploaded to MinIO. + +To exit the ftp client, type `exit`. + +!!! Question "Test the wis2box-ftp service" + + Try to ingest the file `synop.txt` into your wis2box instance using the wis2box-ftp service to trigger the data ingest workflow. + + Check the MinIO user interface to see if the file was uploaded to the correct path in the `wis2box-incoming` bucket. If you don't see the file in MinIO you can check the logs of the wis2box-ftp service to see if there were any errors in the process forwarding the data to MinIO. + + Check the Grafana dashboard to see if the data ingest workflow was triggered or if there were any errors. + +The wis2box-ftp service will forward the data to the MinIO storage service, preserving the directory structure of the uploaded data. To ensure your data is ingested correctly, make sure the file is uploaded to a directory that matches the dataset-id or topic of your dataset. + +## Conclusion + +!!! success "Congratulations!" + In this practical session, you learned how to: + + - trigger wis2box workflow using a Python script and the MinIO Python client + - use different data plugins to ingest different data formats + - forward data to wis2box using the wis2box-ftp service diff --git a/documentation/docs/practical-sessions/automating-data-ingestion.md b/documentation/docs/practical-sessions/automating-data-ingestion.md deleted file mode 100644 index 4fd75976..00000000 --- a/documentation/docs/practical-sessions/automating-data-ingestion.md +++ /dev/null @@ -1,282 +0,0 @@ ---- -title: Data ingest and monitoring ---- - -# Data ingest and monitoring - -!!! abstract "Learning outcomes" - - By the end of this practical session, you will be able to: - - - trigger different wis2box data pipelines using different file types - - monitor the status of your data ingest and publishing - -## Introduction - -The **wis2box-management** container listens to events from the MinIO storage service to trigger data ingestion based on the configuration of ``data-mappings.yml``. This allows you to upload data into MinIO and have wis2box automatically ingest and publish data in real-time. - -For the purpose of next few exercises we will use the MinIO admin interface to upload data into MinIO. - -The same steps can be done programmatically by using any MinIO or S3 client software, allowing you to automate your data ingestion as part of your operational workflows. - -## Preparation - -Login to you student VM using your SSH client (PuTTY or other). - -Make sure wis2box is up and running: - -```bash -cd ~/wis2box-1.0b5/ -python3 wis2box-ctl.py start -python3 wis2box-ctl.py status -``` - -Make sure your have MQTT Explorer running and connected to your instance. -If you are still connected from the previous session, clear any previous messages you may have received from the queue. -This can be done by either by disconnecting and reconnecting or by clicking the trash can for the topic. - -Make sure you have a web browser open with the Grafana dashboard for your instance by going to `http://:3000` - -grafana-homepage - -And make sure you have a second tab open with the MinIO user interface at `http://:9001`. Remember you need to login with the `WIS2BOX_STORAGE_USER` and `WIS2BOX_STORAGE_PASSWORD` defined in your `wis2box.env` file: - -minio-second-tab - -## Ingesting CSV data - -First we will use a test sample in the CSV format using the AWS template. The `data-mappings.yml` on your wis2box are configured to use the `wis2box.data.csv2bufr.ObservationDataCSV2BUFR` plugin for files with the extension `.csv`: - -```{.copy} - csv: - - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR - template: aws-template - notify: true - buckets: - - ${WIS2BOX_STORAGE_INCOMING} - file-pattern: '^.*\.csv$' -``` - -Download the following sample data files to your local machine: - -[aws-example.csv](/sample-data/aws-example.csv) - -Now go back to MinIO in your web browser and navigate to the `wis2box-incoming` bucket and click 'Create new path' to create the following directory: - -`xyz/test/data/core/weather/surface-based-observations/synop` - -minio-admin-create-new-path - -Upload the file `aws-example.csv` to the directory you just created: - -minio-admin-uploaded-file - -!!! question "Exercise 1: check for errors" - Do you see any errors reported on the Grafana dashboard? - -??? success "Click to reveal answer" - The 'wis2box ERRORs' displayed at the bottom of the Grafana home dashboard should report the following error: - - * `ERROR - handle() error: Topic Hierarchy validation error: No plugins for http://minio:9000/wis2box-incoming/xyz/test/data/core/weather/surface-based-observations/synop/aws-example.csv in data mappings. Did not match any of the following: ...` - - Note, the `data` definition in `data-mappings.yml` uses `.` instead of `/` to separate the path elements. - However the path using `.` in the `data-mappings.yml` and `/` in MinIO are equivalent. - - If there are no data mappings defined for the directory that received the data, wis2box will not initiate any workflow. - -!!! question "Exercise 2: correct your input path and repeat the data ingest" - - Go back to MinIO to the root of the `wis2box-incoming` bucket. Then click 'Create new path' and define the correct path for wis2box. For example if your country code is `idn` and your centre-id is `bmkg`, you should create the following path: - - * `idn/bmkg/data/core/weather/surface-based-observations/synop` - - Now upload the sample data file `aws-example.csv` to the new path. Do you see any errors reported on the Grafana dashboard? - -??? success "Click to reveal answer" - The Grafana dashboard should report the following errors: - - * ... {/app/wis2box/data/csv2bufr.py:98} ERROR - Station 0-20000-0-60360 not in station list; skipping - * ... {/app/wis2box/data/csv2bufr.py:98} ERROR - Station 0-20000-0-60355 not in station list; skipping - * ... {/app/wis2box/data/csv2bufr.py:98} ERROR - Station 0-20000-0-60351 not in station list; skipping - - As the stations in the test data are not defined in your wis2box metadata, the data ingest workflow will not be triggered. - - If instead you again see the error `Topic Hierarchy validation error: No plugins for ... in data mappings`, check that you have defined the correct path in MinIO and repeat the data ingest. - -!!! question "Exercise 3: add the test stations and repeat the data ingest" - - Add the following stations to your wis2box using the station editor in **wis2box-webapp**: - - - 0-20000-0-60351 - - 0-20000-0-60355 - - 0-20000-0-60360 - - Now re-upload the sample data file `aws-example.csv` to the same path in MinIO you used in the previous exercise. - - Check the Grafana dashboard, are there any new errors ? How can you see that the test data was successfully ingested and published? - -??? success "Click to reveal answer" - - If you were subscribed with MQTT Explorer to your **wis2box-broker**, you should have received notifications for the test data when the data was successfully published. - - You can also check the charts on the Grafana home dashboard to see if the test data was successfully ingested and published. - - grafana_success - - The chart "Number of WIS2.0 notifications published by wis2box" indicates that notifications were successfully published on the MQTT broker in wis2box. - -## Ingesting binary data - -wis2box can ingest binary data in BUFR format using the `wis2box.data.bufr4.ObservationDataBUFR` plugin included in wis2box. - -You can verify that the plugin is configured in your wis2box by checking the contents of `data-mappings.yml` from the SSH command line: - -```bash -cat ~/wis2box-data/data-mappings.yml -``` - -And you should see it contains an entry that specifies that files with the extension `.bin` should be processed by the `wis2box.data.bufr4.ObservationDataBUFR` plugin: - -```{.copy} - bin: - - plugin: wis2box.data.bufr4.ObservationDataBUFR - notify: true - buckets: - - ${WIS2BOX_STORAGE_INCOMING} - file-pattern: '^.*\.bin$' -``` - -This plugin will split the BUFR file into individual BUFR messages and publish each message to the MQTT broker. If the station for the corresponding BUFR message is not defined in the wis2box station metadata, the message will not be published. - -Please download the following sample data file to your local machine: - -[bufr-example.bin](/sample-data/bufr-example.bin) - -!!! question "Exercise 4: ingest binary data in BUFR format" - - Upload the sample data file 'bufr-example.bin' to the same path in MinIO you used in the previous exercise: - - minio-admin-uploaded-bufr-file - - Check the Grafana dashboard and MQTT Explorer to see if the test-data was successfully ingested and published. - - How many messages were published to the MQTT broker for this data sample? - -??? success "Click to reveal answer" - - If you successfully ingested and published the last data sample, you should have received 10 new notifications on the wis2box MQTT broker. Each notification correspond to data for one station for one observation timestamp. - -## Ingesting SYNOP data in ASCII format - -In the previous session we used the SYNOP form in the **wis2box-webapp** to ingest SYNOP data in ASCII format. You can also ingest SYNOP data in ASCII format by uploading the data into MinIO. The `data-mappings.yml` on your wis2box are configured to use the plugin `wis2box.data.synop2bufr.ObservationDataSYNOP2BUFR` for files with the extension `.txt`: - -```{.copy} - txt: - - plugin: wis2box.data.synop2bufr.ObservationDataSYNOP2BUFR - notify: true - buckets: - - ${WIS2BOX_STORAGE_INCOMING} - file-pattern: '^.*-(\d{4})(\d{2}).*\.txt$' -``` - -Download the following two sample data files to your local machine: - -[synop-202307.txt](/sample-data/synop-202307.txt) - -[synop-202308.txt](/sample-data/synop-202308.txt) - -(click 'save as' in your browser to download the files) - -Note that the 2 files contain the same content, but the filename is different. The filename is used to determine the date of the data sample. - -The file pattern in `data-mappings.yml` specifies that the regular expression `^.*-(\d{4})(\d{2}).*\.txt$` that is used to extract the date from the filename. The first group in the regular expression is used to extract the year and the second group is used to extract the month. - -!!! question "Exercise 5: ingest SYNOP data in ASCII format" - - Go back to the MinIO interface in your browse and navigate to the `wis2box-incoming` bucket and into the path where you uploaded the test data in the previous exercise. - - Upload the new files in the correct path in the `wis2box-incoming` bucket in MinIO to trigger the data ingest workflow. - - Check the Grafana dashboard and MQTT Explorer to see if the test data was successfully ingested and published. - - What is the difference in the `properties.datetime` between the two messages published to the MQTT broker? - -??? success "Click to reveal answer" - - Check the properties of the last 2 notifications in MQTT Explorer and you will note that one notification has: - - ```{.copy} - "properties": { - "data_id": "wis2/rou/test/data/core/weather/surface-based-observations/synop/WIGOS_0-20000-0-60355_20230703T090000", - "datetime": "2023-07-03T09:00:00Z", - ... - ``` - - and the other notification has: - - ```{.copy} - "properties": { - "data_id": "wis2/rou/test/data/core/weather/surface-based-observations/synop/WIGOS_0-20000-0-60355_20230803T090000", - "datetime": "2023-08-03T09:00:00Z", - ... - ``` - - The filename was used to determine the year and month of the data sample. - -## MinIO Python client (optional) - -In this exercise we will use the MinIO Python client to copy data into MinIO. - -MinIO provides a Python client which can be installed as follows: - -```bash -pip3 install minio -``` - -On your student VM the 'minio' package for Python will already be installed. - -Go to the directory `exercise-materials/data-ingest` and run the example script using the following command: - -```bash -cd ~/exercise-materials/data-ingest -python3 copy_data_to_incoming.py -``` - -!!! note - - You will get an error as the script is not configured to access the MinIO endpoint on your wis2box yet. - -The script needs to know the correct endpoint for accessing MinIO on your wis2box. If wis2box is running on your host, the MinIO endpoint is available at `http://:9000`. - -The sample script provides the basic structure for copying a file into MinIO. - -!!! question "ingest data using Python" - Use the Python example provided to create to ingest data into your wis2box. - - Ensure that you: - - - define the correct MinIO endpoint for your host - - define the correct path in MinIO for the topics defined in your `data-mappings.yml` - - provide the correct storage credentials for your MinIO instance - -You can verify that the data was uploaded correctly by checking the MinIO user interface and seeing if the sample data is available in the correct directory in the `wis2box-incoming` bucket. - -You can use the Grafana dashboard to check the status of the data ingest workflow. - -Finally you can use MQTT Explorer to check if notifications were published for the data you ingested. - -## Cleaning up - -You can now delete the following stations you have created in your wis2box using the station-editor in **wis2box-webapp**: - -- 0-20000-0-60351 -- 0-20000-0-60355 -- 0-20000-0-60360 - -## Conclusion - -!!! success "Congratulations!" - In this practical session, you learned how to: - - - trigger wis2box workflow using different data ingest methods - - monitor the status of your data ingest and publishing diff --git a/documentation/docs/practical-sessions/bufr-command-line-tools.md b/documentation/docs/practical-sessions/bufr-command-line-tools.en.md similarity index 94% rename from documentation/docs/practical-sessions/bufr-command-line-tools.md rename to documentation/docs/practical-sessions/bufr-command-line-tools.en.md index 003e16bc..d13cbe56 100644 --- a/documentation/docs/practical-sessions/bufr-command-line-tools.md +++ b/documentation/docs/practical-sessions/bufr-command-line-tools.en.md @@ -1,41 +1,42 @@ --- -title: BUFR command line tools +title: Working with BUFR data --- -# BUFR command line tools +# Working with BUFR data !!! abstract "Learning outcomes" - In this practical session you will be introduced to some of the BUFR command line tools included in the **wis2box-management** container. You will learn: + In this practical session you will be introduced to some of the BUFR tools included in the **wis2box-api** container that are used to transform data to BUFR format and to read the content encoded in BUFR. + + You will learn: - how to inspect the headers in the BUFR file using the `bufr_ls` command - how to extract and inspect the data within a bufr file using `bufr_dump` - - the basic structure of the bufr templats used in csv2bufr and how to use the command line tool + - the basic structure of the bufr templates used in csv2bufr and how to use the command line tool - and how to make basic changes to the bufr templates and how to update the wis2box to use the revised version ## Introduction -The wis2box-management container contains some tools for working with BUFR files from the command line. +The plugins that produces notifications with BUFR data use processes in the wis2box-api to work with BUFR data, for example to transform the data from CSV to BUFR or from BUFR to geojson. + +The wis2box-api container includes a number of tools to work with BUFR data. These include the tools developed by ECMWF and included in the ecCodes software, more information on these can be found on the [ecCodes website](https://confluence.ecmwf.int/display/ECC/BUFR+tools). -Other tools include thosedeveloped as part of the wis2box development, including csv2bufr and synop2bufr that you have previously used -but via the wis2box web-application. - In this session you will be introduced to the `bufr_ls` and `bufr_dump` from the ecCodes software package and advanced configuration of the csv2bufr tool. ## Preparation -In order to use the BUFR command line tools you will need to be logged in to the wis2box management container +In order to use the BUFR command line tools you will need to be logged in to the wis2box-api container and unless specified otherwise all commands should be run on this container. You will also need to have MQTT Explorer open and connected to your broker. -First connect to your student VM via your ssh client and then log in the to management container: +First connect to your student VM via your ssh client and then log in the to wis2box-api container: ```{.copy} -cd ~/wis2box-1.0b5 -python3 wis2box-ctl.py login +cd ~/wis2box-1.0b8 +python3 wis2box-ctl.py login wis2box-api ``` Confirm the that tools are available, starting with ecCodes: @@ -549,36 +550,6 @@ csv2bufr data transform --bufr-template aws-template-custom csv2bufr-ex1.csv Inspect the output file using `bufr_ls` and confirm that the originating centre in the headers is updated. -!!! note - - You can use the custom mappings in your automated workflow by updating your data-mappings.yml and updating the environment variable for CSV2BUFR_TEMPLATES. - - Edit your `data-mappings.yml` file to use this new file by updating the template name: - - ``` - csv: - - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR - template: aws-template-custom - notify: true - file-pattern: '^.*\.csv$' - ``` - - Add the environment variable to your `wis2box.env` file: - - ```{.copy} - cd ~/wis2box-1.0b5/ - echo "export CSV2BUFR_TEMPLATES=/data/wis2box/bufr-templates" >> wis2box.env - ``` - - And restart the wis2box stack: - - ```{.copy} - python3 wis2box-ctl.py stop - python3 wis2box-ctl.py start - ``` - - The new template should now be used in the automated workflow. - ## Housekeeping Clean up your working directory by removing files you no longer need and clean up your station list to remove any diff --git a/documentation/docs/practical-sessions/configuring-station-metadata.md b/documentation/docs/practical-sessions/configuring-station-metadata.en.md similarity index 57% rename from documentation/docs/practical-sessions/configuring-station-metadata.md rename to documentation/docs/practical-sessions/configuring-station-metadata.en.md index 1a1c2388..ee9b36de 100644 --- a/documentation/docs/practical-sessions/configuring-station-metadata.md +++ b/documentation/docs/practical-sessions/configuring-station-metadata.en.md @@ -10,23 +10,22 @@ title: Configuring station metadata - create an authorization token for the `collections/stations` endpoint - add station metadata to wis2box - - review stations associated to datasets in the **wis2box-ui** - update/delete station metadata using the **wis2box-webapp** ## Introduction -wis2box has a collection of station metadata that is used to publish data on WIS2. -Only data for stations configured in the wis2box station list will be published on your wis2box broker. -The **WIGOS Station Identifier (WSI)** is used as the unique reference of the station which produced a specific set of observation data. +For sharing data internationally between WMO Members, it is important to have a common understanding of the stations that are producing the data. The WMO Integrated Global Observing System (WIGOS) provides a framework for the integration of observing systems and data management systems. The **WIGOS Station Identifier (WSI)** is used as the unique reference of the station which produced a specific set of observation data. + +wis2box has a collection of station metadata that is used to describe the stations that are producing the observation data and should be retrieved from **OSCAR/Surface**. The station metadata in wis2box is used by the BUFR transformation tools to check that input data contains a valid WIGOS Station Identifier (WSI) and to provide a mapping between the WSI and the station metadata. ## Create an authorization token for collections/stations To edit stations via the **wis2box-webapp** you will first to need create an authorization token. -Login to your student VM and ensure you are in the `wis2box-1.0b5` directory: +Login to your student VM and ensure you are in the `wis2box-1.0b8` directory: ```bash -cd ~/wis2box-1.0b5 +cd ~/wis2box-1.0b8 ``` Then login into the **wis2box-management** container with the following command: @@ -63,9 +62,7 @@ Continue with token: DataIsMagic [y/N]? y Token successfully created ``` -!!! note "Exercise 1: Create an authorization token for collections/stations" - - Please create an authorization token for the `collections/stations` endpoint using the instructions above. +Please create an authorization token for the `collections/stations` endpoint using the instructions above. ## add station metadata using the **wis2box-webapp** @@ -83,6 +80,17 @@ When you click add 'add new station' you are asked to provide the WIGOS station wis2box-webapp-import-station-from-oscar +!!! note "Add station metadata for 3 or more stations" + Please add three or more stations to the wis2box station metadata collection of your wis2box. + + Please use stations from your country if possible, especially if you brought your own data. + + If your country does not have any stations in OSCAR/Surface, you can use the following stations for the purpose of this exercise: + + - 0-20000-0-91334 + - 0-20000-0-96323 (note missing station elevation in OSCAR) + - 0-20000-0-96749 (note missing station elevation in OSCAR) + When you click search the station data is retrieved from OSCAR/Surface, please note that this can take a few seconds. Review the data returned by OSCAR/Surface and add missing data where required. Select a topic for the station and provide your authorization token for the `collections/stations` endpoint and click 'save': @@ -95,26 +103,16 @@ Go back to the station list and you will see the station you added: wis2box-webapp-stations-with-one-station -!!! note "Exercise 2: Add station metadata" - - Please add three or more stations to the wis2box station metadata collection of your wis2box. - - Please use stations from your country if possible, especially if you brought your own data. - - Otherwise, you can use the following WIGOS station identifiers for testing purposes: - - - 0-20000-0-91334 - - 0-20000-0-96323 (note missing station elevation in OSCAR) - - 0-20000-0-96749 (note missing station elevation in OSCAR) +Repeat this process until you have at least 3 stations configured. !!! tip "Deriving missing elevation information" If your station elevation is missing, there are online services to help lookup the elevation using open elevation data. One such example is the [Open Topo Data API](https://www.opentopodata.org). - For example, to get the elevation of the BMKG auditorium, one would query the Open Topo Data API as follows: + For example, to get the elevation at latitude -6.15558 and longitude 106.84204, you can copy-paste the following URL in a new browser-tab: - ```bash - wget -q -O - "https://api.opentopodata.org/v1/aster30m?locations=-6.15558,106.84204" + ```{.copy} + https://api.opentopodata.org/v1/aster30m?locations=-6.15558,106.84204 ``` Output: @@ -137,36 +135,44 @@ Go back to the station list and you will see the station you added: ## Review your station metadata -After saving your station metadata, you can review the content of your station metadata in the **wis2box-webapp**: +The station metadata is stored in the backend of wis2box and made available via the **wis2box-api**. -You can verify the updated stations are available in the **wis2box-api**: +If you open a browser and navigate to `http:///oapi/collections/stations/items` you will see the station metadata you added: wis2box-api-stations -You can also visit the **wis2box-ui** at `http://` and select "EXPLORE" on your dataset and you will see the stations you added: +!!! note "Review your station metadata" -wis2box-ui-explore-stations - -!!! note "Exercise 3: Review your station metadata" - - Verify the stations you added are associated to your dataset by visiting the **wis2box-api** and **wis2box-ui** endpoints for your host in your browser. + Verify the stations you added are associated to your dataset by visiting `http:///oapi/collections/stations/items` in your browser. You also have the option to view/update/delete the station in the **wis2box-webapp**. Note that you are required to provide your authorization token for the `collections/stations` endpoint to update/delete the station. -!!! note "Exercise 4: Update/delete station metadata" +!!! note "Update/delete station metadata" - Please update/delete the station metadata for one of the stations you added using the **wis2box-webapp**. + Try and see if you can update/delete the station metadata for one of the stations you added using the **wis2box-webapp**. ## Bulk station metadata upload -wis2box also has the ability to perform "bulk" loading of station metadata from a CSV file. See the official [wis2box documentation](https://docs.wis2box.wis.wmo.int/en/1.0b5/reference/running/station-metadata.html) for more information. +Note that wis2box also has the ability to perform "bulk" loading of station metadata from a CSV file using the command line in the **wis2box-management** container. + +```bash +python3 wis2box-ctl.py login +wis2box metadata station publish-collection -p /data/wis2box/metadata/station/station_list.csv -th origin/a/wis2/centre-id/weather/surface-based-observations/synop +``` + +This allows you to upload a large number of stations at once and associate them with a specific topic. + +You can create the CSV file using Excel or a text editor and then upload it to wis2box-host-datadir to make it available to the **wis2box-management** container in the `/data/wis2box/` directory. + +After doing a bulk upload of stations, it is recommended to review the stations in the **wis2box-webapp** to ensure the data was uploaded correctly. + +See the official [wis2box documentation](https://docs.wis2box.wis.wmo.int) for more information on how to use this feature. ## Conclusion !!! success "Congratulations!" In this practical session, you learned how to: - - create an authorization token for the `collections/stations` endpoint - - add station metadata to wis2box - - review stations associated to datasets in the **wis2box-ui** - - update/delete station metadata using the **wis2box-webapp** + - create an authorization token for the `collections/stations` endpoint to be used with the **wis2box-webapp** + - add station metadata to wis2box using the **wis2box-webapp** + - view/update/delete station metadata using the **wis2box-webapp** diff --git a/documentation/docs/practical-sessions/configuring-wis2-discovery-metadata.md b/documentation/docs/practical-sessions/configuring-wis2-discovery-metadata.md deleted file mode 100644 index 119a0bab..00000000 --- a/documentation/docs/practical-sessions/configuring-wis2-discovery-metadata.md +++ /dev/null @@ -1,153 +0,0 @@ ---- -title: Configuring WIS2 discovery metadata ---- - -# Configuring WIS2 discovery metadata - -!!! abstract "Learning outcomes" - By the end of this practical session, you will be able to: - - - create discovery metadata - - publish discovery metadata - - update and re-publish discovery metadata - -## Introduction - -WIS2 requires discovery metadata to be provided describing -your data to be shared to WIS2 Global Services. This session will walk you through creating and publishing -discovery metadata from wis2box from a configuration file. - -!!! note - While this practical session uses discovery metadata to publish SYNOP, feel free to use another dataset (such as TEMP). - - The discovery metadata configuration file for TEMP can be found in `~/wis2box-data/metadata/discovery/metadata-temp.yml`. - - -## Preparation - -Ensure you are running MQTT Explorer and you are connected to the broker on your student VM before continuing. - -Login to your student VM using your SSH-client. - -## Creating discovery metadata - -Let's start by using the file generated during the [Initializing wis2box](../initializing-wis2box) practical session. - -Inspect the sample SYNOP discovery metadata: - -```bash -more ~/wis2box-data/metadata/discovery/metadata-synop.yml -``` - -!!! note - All values in the discovery metadata configuration are required and should be included. - -!!! question - How does line 3 of your discovery metadata file relate to the new data mapping in the previous session? - -??? success "Click to reveal answer" - Line 3 of the discovery metadata configuration file should be equal to one of the data mappings defined in `data-mappings.yml`. - -Update the following values in the discovery metadata configuration: - -- `identification.title`: a human readable title describing your data -- `identification.abstract`: a human readable description describing your data -- `identification.extents.temporal (begin)`: the begin (`BEGIN_DATE`) (`YYYY-MM-DD`) and end time of your data (keeping the end time to `null` is suitable to ongoing observations) -- `contact.pointOfContact`: your organization's point of contact information - -!!! note - Many values have been pre-populated as part of the [Initializing wis2box](../initializing-wis2box) practical session, however you will want to ensure they are correct / accurate for your dataset. - -!!! tip - The configuration is based on the YAML format. Consult the [YAML cheatsheet](../cheatsheets/yaml.md) for more information. - -!!! tip - As the `identification.extents.spatial.bbox` coordinates are pre-populated, you may want to update the coordinates if the dataset coordinates are not based on your country's administrative boundaries. Feel free to update this value accordingly, ensuring that values are correctly signed (for example, use the minus sign [`-`] for southern or western hemispheres. - - The following tools can be valuable for deriving your `identification.extents.spatial.bbox`: - - - [https://gist.github.com/graydon/11198540](https://gist.github.com/graydon/11198540) - - [http://bboxfinder.com](http://bboxfinder.com) - - [https://boundingbox.klokantech.com](https://boundingbox.klokantech.com) - -## Publishing discovery metadata - -First login to the **wis2box-management** container: - -```bash -python3 wis2box-ctl.py login -``` - -Run the following command to publish your discovery metadata: - -```bash -wis2box metadata discovery publish /data/wis2box/metadata/discovery/metadata-synop.yml -``` - -Ensure that your discovery metadata was published to the API, by navigating to `http:///oapi/collections/discovery-metadata`. - -Ensure that your discovery metadata was also published to the broker, by looking for a new metadata message in MQTT Explorer. - -!!! question - Do you see your new discovery metadata in the API? - -??? success "Click to reveal answer" - You should see your published discovery metadata in the API, under the `http:///oapi/collections/discovery-metadata/items` link. - -Click on your discovery metadata record and inspect the content, noting how it relates to the discovery metadata configuration created earlier in this session. - -Update the title of your discovery metadata: - -```bash -vi /data/wis2box/metadata/discovery/metadata-synop.yml -``` - -!!! tip - You can also use WinSCP to connect to your instance and edit this file. - -Now re-publish (from inside the **wis2box-management** container): - -```bash -wis2box metadata discovery publish /data/wis2box/metadata/discovery/metadata-synop.yml -``` - -Ensure that your discovery metadata updates were published to the API, by refreshing the page to your discovery metadata. - -!!! question - Are you able to see the updates you made in the configuration? - -??? success "Click to reveal answer" - You should see your published discovery metadata update in the API, under the `http:///oapi/collections/discovery-metadata/items` link. - -Feel free to update additional values and re-publishing your discovery metadata to get a better idea of how and where discovery metadata content is updated. - -## Publishing your dataset to the API - -Run the below command to add the data to the API: - -```bash -wis2box data add-collection /data/wis2box/metadata/discovery/metadata-synop.yml -``` - -Ensure that your dataset was published to the API, by navigating to `http:///oapi/collections/`. - -!!! question - Do you see your new dataset in the API? - -??? success "Click to reveal answer" - You should see the new dataset published to the API. - -!!! question - Do you see any data coming from your new dataset in the API? If not, why not? - -??? success "Click to reveal answer" - If you do not see new data, this means that data has not been ingested yet against your new dataset. - -## Conclusion - -!!! success "Congratulations!" - In this practical session, you learned how to: - - - create discovery metadata - - publish discovery metadata - - update and re-publish discovery metadata diff --git a/documentation/docs/practical-sessions/configuring-wis2box-datasets.en.md b/documentation/docs/practical-sessions/configuring-wis2box-datasets.en.md new file mode 100644 index 00000000..a5ef0a09 --- /dev/null +++ b/documentation/docs/practical-sessions/configuring-wis2box-datasets.en.md @@ -0,0 +1,199 @@ +--- +title: Configuring datasets in wis2box +--- + +# Configuring datasets in wis2box + +!!! abstract "Learning outcomes" + By the end of this practical session, you will be able to: + + - create a new dataset + - create discovery metadata for a dataset + - configure data mappings for a dataset + - publish a WIS2 notification with a WCMP2 record + - update and re-publish your dataset + +## Introduction + +wis2box uses datasets that are associated with discovery metadata and data mappings. + +Discovery metadata is used to create a WCMP2 (WMO Core Metadata Profile 2) record that is shared using a WIS2 notification published on your wis2box-broker. + +The data mappings are used to associate a data plugin to your input data, allowing your data to be transformed prior to being published using the WIS2 notification. + +This session will walk you through creating a new dataset, creating discovery metadata, and configuring data mappings. You will inspect your dataset in the wis2box-api and review the WIS2 notification for your discovery metadata. + +## Preparation + +Connect to your broker using MQTT Explorer. + +Instead of using your internal broker credentials, use the public credentials `everyone/everyone`: + +MQTT Explorer: Connect to broker + +!!! Note + + You never need to share the credentials of your internal broker with external users. The 'everyone' user is a public user to enable sharing of WIS2 notifications. + + The `everyone/everyone` credentials has read-only access on the topic 'origin/a/wis2/#'. This is the topic where the WIS2 notifications are published. The Global Broker can subscribe with these public credentials to receive the notifications. + + The 'everyone' user will not see internal topics or be able to publish messages. + +Open a browser and open a page to `http:///wis2box-webapp`. Make sure you are logged in and can access the 'dataset editor' page. + +See the section on [Initializing wis2box](/practical-sessions/initializing-wis2box) if you need to remember how to connect to the broker or access the wis2box-webapp. + +## Create an authorization token for collections/stations + +You will need an authorization token for the 'processes/wis2box' endpoint to publish your dataset. + +To create an authorization token, access your training VM over SSH and use the following commands to login to the wis2box-management container: + +```bash +cd ~/wis2box-1.0b8 +python3 wis2box-ctl.py login +``` + +Then run the following command to create a randomly generated authorization token for the 'processes/wis2box' endpoint: + +```bash +wis2box auth add-token --path processes/wis2box +``` + +You can also create a token with a specific value by providing the token as an argument to the command: + +```bash +wis2box auth add-token --path processes/wis2box MyS3cretToken +``` + +Make sure to copy the token value and store it on your local machine, as you will need it later. + +Once you have your token, you can exit the wis2box-management container: + +```bash +exit +``` + +## Creating a new dataset in the wis2box-webapp + +Navigate to the 'dataset editor' page in the wis2box-webapp of your wis2box instance by going to `http:///wis2box-webapp` and selecting 'dataset editor' from the menu on the left hand side. + +On the 'dataset editor' page, under the 'Datasets' tab, click on "Create New ...": + +Create New Dataset + +A pop-up window will appear, asking you to provide: + +- **Centre ID** : this is the agency acronym (in lower case and no spaces), as specified by the WMO Member, that identifies the data centre responsible for publishing the data. +- **Data Type**: The type of data you are creating metadata for. You can choose between using a predefined template or selecting 'other'. If 'other' is selected, more fields will have to be manually filled. + +!!! Note + + Your centre-id should start with the TLD of your country, followed by a dash (`-`) and an abbreviated name of your organization (for example `fr-meteofrance`). The centre-id must be lowercase and use alphanumeric characters only. The dropdown list shows all currently registered centre-ids on WIS2 as well as any centre-id you have already created in wis2box. + +Please choose a centre-id appropriate for your organization. + +For **Data Type**, select **weather/surface-based-observations/synop**: + +Create New Dataset Form: Initial information + +Click *continue to form* to proceed, you will now be presented with the **Dataset Editor Form**. + +Since you selected the **weather/surface-based-observations/synop** data type, the form will be pre-populated with some initial values related to this data type. + +## Creating discovery metadata + +The Dataset Editor Form allows you to provide the Discovery Metadata for your dataset that the wis2box-management container will use to publish a WCMP2 record. + +Since you have selected the 'weather/surface-based-observations/synop' data type, the form will be pre-populated with some default values. + +Review the title and keywords, and update them as necessary, and provide a description for your dataset: + +Metadata Editor: title, description, keywords + +Note there are options to change the 'WMO Data Policy' from 'core' to 'recommended' or to modify your default Metadata Identifier, please keep data-policy as 'core' and use the default Metadata Identifier. + +Next, review the section defining your 'Temporal Properties' and 'Spatial Properties'. You can adjust the bounding box by updating the 'North Latitude', 'South Latitude', 'East Longitude', and 'West Longitude' fields: + +Metadata Editor: temporal properties, spatial properties + +Next, fill out the section defining the 'Contact Information of the Data Provider': + +Metadata Editor: contact information + +Finally, fill out the section defining the 'Data Quality Information': + +Once you are done filling out all the sections, click 'VALIDATE FORM' and check the form for any errors: + +Metadata Editor: validation + +If there are any errors, correct them and click 'VALIDATE FORM' again. + +Making sure you have no errors and that you get a pop-up indication your form has been validated: + +Metadata Editor: validation success + +Next, before submitting your dataset, review the data mappings for your dataset. + +## Configuring data mappings + +Since you used a template to create your dataset, the dataset mappings have been pre-populated with the defaults plugins for the 'weather/surface-based-observations/synop' data type. Data plugins are used in the wis2box to transform data before it is published using the WIS2 notification. + +Data Mappings: update plugin + +Note that you can click on the "update"-button to change settings for the plugin such as file-extension and the file-pattern, you can leave the default settings for now. In a later session, you will learn more about BUFR and the transformation of data into BUFR format. + +## Submitting your dataset + +Finally, you can click 'submit' to publish your dataset. + +You will need to provide the authorization token for 'processes/wis2box' that you created earlier. If you have not done so, you can create a new token by following the instructions in the preparation section. + +Check that you get the following message after submitting your dataset, indicating that the dataset was successfully submitted: + +Submit Dataset Success + +After you click 'OK', you are redirected to the Dataset Editor home page. Now if you click on the 'Dataset' tab, you should see your new dataset listed: + +Dataset Editor: new dataset + +## Reviewing the WIS2-notification for your discovery metadata + +Go to MQTT Explorer, if you were connected to the broker, you should see a new WIS2 notification published on the topic `origin/a/wis2//metadata`: + +MQTT Explorer: WIS2 notification + +Inspect the content of the WIS2 notification you published. You should see a JSON with a structure corresponding to the WIS Notification Message (WNM) format. + +!!! question + + On what topic is the WIS2 notification published? + +??? success "Click to reveal answer" + + The WIS2 notification is published on the topic `origin/a/wis2//metadata`. + +!!! question + + Try to find the title, description and keywords you provided in the discovery metadata in the WIS2 notification. Can you find them? + +??? success "Click to reveal answer" + + Note that the title, description, and keywords you provided in the discovery metadata are **not** present in the WIS2 notification payload! + + Instead, try to look for the canonical link in the "links"-section in the WIS2 notification: + + WIS2 notification for metadata, links sections + + The WIS2 notification contains a canonical link to the WCMP2 record that was published. If you copy-paste this link into a browser, you will download the WCMP2 record and see the title, description, and keywords you provided. + +## Conclusion + +!!! success "Congratulations!" + In this practical session, you learned how to: + + - create a new dataset + - define your discovery metadata + - review your data mappings + - publish discovery metadata + - review the WIS2 notification for your discovery metadata diff --git a/documentation/docs/practical-sessions/connecting-to-wis2-over-mqtt.en.md b/documentation/docs/practical-sessions/connecting-to-wis2-over-mqtt.en.md new file mode 100644 index 00000000..bcae48e6 --- /dev/null +++ b/documentation/docs/practical-sessions/connecting-to-wis2-over-mqtt.en.md @@ -0,0 +1,200 @@ +--- +title: Connecting to WIS2 over MQTT +--- + +# Connecting to WIS2 over MQTT + +!!! abstract "Learning outcomes" + + By the end of this practical session, you will be able to: + + - connect to the WIS2 Global Broker using MQTT Explorer + - review the WIS2 topic structure + - review the WIS2 notification message structure + +## Introduction + +WIS2 uses the MQTT protocol to advertise the availability of weather/climate/water data. The WIS2 Global Broker subscribes to all WIS2 Nodes in the network and republishes the messages it receives. The Global Cache subscribes to the Global Broker, downloads the data in the message and then republishes the message on the `cache` topic with a new URL. The Global Discovery Catalogue publishes discovery metadata from the Broker and provides a search API. + +This is an example of the WIS2 notification message structure for a message received on the topic `origin/a/wis2/br-inmet/data/core/weather/surface-based-observations/synop`: + +```json +{ + "id": "59f9b013-c4b3-410a-a52d-fff18f3f1b47", + "type": "Feature", + "version": "v04", + "geometry": { + "coordinates": [ + -38.69389, + -17.96472, + 60 + ], + "type": "Point" + }, + "properties": { + "data_id": "br-inmet/data/core/weather/surface-based-observations/synop/WIGOS_0-76-2-2900801000W83499_20240815T060000", + "datetime": "2024-08-15T06:00:00Z", + "pubtime": "2024-08-15T09:52:02Z", + "integrity": { + "method": "sha512", + "value": "TBuWycx/G0lIiTo47eFPBViGutxcIyk7eikppAKPc4aHgOmTIS5Wb9+0v3awMOyCgwpFhTruRRCVReMQMp5kYw==" + }, + "content": { + "encoding": "base64", + "value": "QlVGUgAA+gQAABYAACsAAAAAAAIAHAAH6AgPBgAAAAALAAABgMGWx1AAAM0ABOIAAAODM0OTkAAAAAAAAAAAAAAKb5oKEpJ6YkJ6mAAAAAAAAAAAAAAAAv0QeYA29WQa87ZhH4CQP//z+P//BD////+ASznXuUb///8MgAS3/////8X///e+AP////AB/+R/yf////////////////////6/1/79H/3///gEt////////4BLP6QAf/+/pAB//4H0YJ/YeAh/f2///7TH/////9+j//f///////////////////v0f//////////////////////wNzc3Nw==", + "size": 250 + }, + "wigos_station_identifier": "0-76-2-2900801000W83499" + }, + "links": [ + { + "rel": "canonical", + "type": "application/x-bufr", + "href": "http://wis2bra.inmet.gov.br/data/2024-08-15/wis/br-inmet/data/core/weather/surface-based-observations/synop/WIGOS_0-76-2-2900801000W83499_20240815T060000.bufr4", + "length": 250 + } + ] +} +``` + +In this practical session you will learn how to use the MQTT Explorer tool to setup an MQTT client connection to a WIS2 Global Broker and be able to display WIS2 notification messages. + +MQTT Explorer is a useful tool to browse and review the topic structure for a given MQTT broker to review data being published. + +Note that MQTT is primarily used for "machine-to-machine" communication; meaning that there would normally be a client automatically parsing the messages as they are received. To work with MQTT programmatically (for example, in Python), you can use MQTT client libraries such as [paho-mqtt](https://pypi.org/project/paho-mqtt) to connect to an MQTT broker and process incoming messages. There exist numerous MQTT client and server software, depending on your requirements and technical environment. + +## Using MQTT Explorer to connect to the Global Broker + +To view messages published by a WIS2 Global Broker you can "MQTT Explorer" which can be downloaded from the [MQTT Explorer website](https://mqtt-explorer.com). + +Open MQTT Explorer and add a new connection to the Global Broker hosted by MeteoFrance using the following details: + +- host: globalbroker.meteo.fr +- port: 8883 +- username: everyone +- password: everyone + +mqtt-explorer-global-broker-connection + +Click on the 'ADVANCED' button, remove the pre-configured topics and add the following topics to subscribe to: + +- `origin/#` + +mqtt-explorer-global-broker-advanced + +!!! note + When setting up MQTT subscriptions you can use the following wildcards: + + - **Single-level (+)**: a single-level wildcard replaces one topic level + - **Multi-level (#)**: a multi-level wildcard replaces multiple topic levels + + In this case `origin/#` will subscribe to all topics under the `origin` topic. + +Click 'BACK', then 'SAVE' to save your connection and subscription details. Then click 'CONNECT': + +Messages should start appearing in your MQTT Explorer session as follows: + +mqtt-explorer-global-broker-topics + +You are now ready to start exploring the WIS2 topics and message structure. + +## Exercise 1: Review the WIS2 topic structure + +Use MQTT to browse topic structure under the `origin` topics. + +!!! question + + How can we distinguish the WIS centre that published the data? + +??? success "Click to reveal answer" + + You can click on the left hand side window in MQTT Explorer to expand the topic structure. + + We can distinguish the WIS centre that published the data by looking at the fourth level of the topic structure. For example, the following topic: + + `origin/a/wis2/br-inmet/data/core/weather/surface-based-observations/synop` + + tells us that the data was published a WIS centre with the centre-id `br-inmet`, which is the centre-id for Instituto Nacional de Meteorologia - INMET, Brazil. + +!!! question + + How can we distinguish between messages published by WIS-centres hosting a GTS-to-WIS2 gateway and messages published by WIS-centres hosting a WIS2 node? + +??? success "Click to reveal answer" + + We can distinguish messages coming from GTS-to-WIS2 gateway by looking at the centre-id in the topic structure. For example, the following topic: + + `origin/a/wis2/de-dwd-gts-to-wis2/data/core/I/S/A/I/01/sbbr` + + tells us that the data was published by the GTS-to-WIS2 gateway hosted by Deutscher Wetterdienst (DWD), Germany. The GTS-to-WIS2 gateway is a special type of data-publisher that publishes data from the Global Telecommunication System (GTS) to WIS2. The topic structure is composed by the TTAAii CCCC headers for the GTS messages. + +## Exercise 2: Review the WIS2 message structure + +Disconnect from MQTT Explorer and update the 'Advanced' sections to change the subscription to the following: + +* `origin/a/wis2/+/data/core/weather/surface-based-observations/synop` +* `cache/a/wis2/+/data/core/weather/surface-based-observations/synop` + +mqtt-explorer-global-broker-topics-exercise2 + +!!! note + The `+` wildcard is used to subscribe to all WIS-centres. + +Reconnect to the Global Broker and wait for messages to appear. + +You can view the content of the WIS2 message in the "Value" section on the right hand side. Try to expand the topic structure to see the different levels of the message until you reach the last level and review message content of one of the messages. + +!!! question + + How can we identify the timestamp that the data was published? And how can we identify the timestamp that the data was collected? + +??? success "Click to reveal answer" + + The timestamp that the data was published is contained in the `properties` section of the message with a key of `pubtime`. + + The timestamp that the data was collected is contained in the `properties` section of the message with a key of `datetime`. + + mqtt-explorer-global-broker-msg-properties + +!!! question + + How can we download the data from the URL provided in the message? + +??? success "Click to reveal answer" + + The URL is contained in the `links` section with `rel="canonical"` and defined by the `href` key. + + You can copy the URL and paste it into a web browser to download the data. + +## Exercise 3: Review the difference between 'origin' and 'cache' topics + +Make sure you are still connected to the Global Broker using the topic subscriptions `origin/a/wis2/+/data/core/weather/surface-based-observations/synop` and `cache/a/wis2/+/data/core/weather/surface-based-observations/synop` as described in Exercise 2. + +Try to identify a message for the same centre-id published on both the `origin` and `cache` topics. + + +!!! question + + What is the difference between the messages published on the `origin` and `cache` topics? + +??? success "Click to reveal answer" + + The messages published on the `origin` topics are the original messages which the Global Broker republishes from the WIS2 Nodes in the network. + + The messages published on the `cache` topics are the messages for data has been downloaded by the Global Cache. If you check the content of the message from the topic starting with `cache`, you will see that the 'canonical' link has been updated to a new URL. + + There are multiple Global Caches in the WIS2 network, so you will receive one message from each Global Cache that has downloaded the message. + + The Global Cache will only download and republish messages that were published on the `../data/core/...` topic hierarchy. + +## Conclusion + +!!! success "Congratulations!" + In this practical session, you learned: + + - how to subscribe to WIS2 Global Broker services using MQTT Explorer + - the WIS2 topic structure + - the WIS2 notification message structure + - the difference between core and recommended data + - the topic structure used by the GTS-to-WIS2 gateway + - the difference between Global Broker messages published on the `origin` and `cache` topics diff --git a/documentation/docs/practical-sessions/connecting-to-wis2-over-mqtt.md b/documentation/docs/practical-sessions/connecting-to-wis2-over-mqtt.md deleted file mode 100644 index a1160e5d..00000000 --- a/documentation/docs/practical-sessions/connecting-to-wis2-over-mqtt.md +++ /dev/null @@ -1,189 +0,0 @@ ---- -title: Connecting to WIS2 over MQTT ---- - -# Connecting to WIS2 over MQTT - -!!! abstract "Learning outcomes" - - By the end of this practical session, you will be able to: - - - connect to the WIS2 Global Broker using MQTT Explorer - - review the WIS2 topic structure - - review the WIS2 notification message structure - -## Introduction - -WIS2 uses the MQTT protocol to advertise the availability of weather/climate/water data. The WIS2 Global Broker subscribes to all WIS2 Nodes in the network and republishes the messages it receives. The Global Cache subscribes to the Global Broker, downloads the data in the message and then republishes the message on the `cache` topic with a new URL. The Global Discovery Catalogue publishes discovery metadata from the Broker and provides a search API. - -As part of the WIS2 Pilot Phase in 2023, two Global Brokers are available (hosted by China Meteorological Administration and Météo-France). - -This is an example of the WIS2 notification message structure for a message received on the topic `origin/a/wis2/arg/sabm/data/core/weather/surface-based-observations/synop`: - -```json -{ - "id": "fa587559-b02e-40a2-9fd5-2c141c39b130", - "type": "Feature", - "version": "v04", - "geometry": { - "type": "Point", - "coordinates": [ - -56.625, - -64.24139, - 208 - ] - }, - "properties": { - "data_id": "wis2/arg/sabm/data/core/weather/surface-based-observations/synop/WIGOS_0-20000-0-89055_20230926T190000", - "datetime": "2023-09-26T19:00:00Z", - "pubtime": "2023-09-26T18:48:51Z", - "integrity": { - "method": "sha512", - "value": "5a239b5d2ba7a04bd3b2fa44a73a9fb98167d0d4424d7fd19c0a83c75a7715212e2b97b5db6581bb3e1895b92232614d4cc4841ee9164baf47183c8403668dbd" - }, - "wigos_station_identifier": "0-20000-0-89055" - }, - "links": [ - { - "rel": "canonical", - "type": "application/x-bufr", - "href": "http://w2b.smn.gov.ar/data/2023-09-26/wis/arg/sabm/data/core/weather/surface-based-observations/synop/WIGOS_0-20000-0-89055_20230926T190000.bufr4", - "length": 254 - }, - { - "rel": "via", - "type": "text/html", - "href": "https://oscar.wmo.int/surface/#/search/station/stationReportDetails/0-20000-0-89055" - } - ] -} -``` - -In this practical session you will learn how to use the MQTT Explorer tool to review the topics available on this Global Broker and be able to display WIS2 notification messages. - -MQTT Explorer is a helpful tool to browse and review the topic structure for a given MQTT broker and visually interact with the MQTT protocol. There exist numerous other MQTT client and server software, depending on your requirements and technical environment. - -To work with MQTT programmatically (for example, in Python), you can use MQTT client libraries such as [paho-mqtt](https://pypi.org/project/paho-mqtt) to connect to an MQTT broker and process incoming messages. - -## Using MQTT Explorer to connect to the Global Broker - -One way to view messages published by this Global Broker is using the MQTT Explorer which can be downloaded from the [MQTT Explorer website](https://mqtt-explorer.com). - -Open MQTT Explorer and add a new connection to the Global Broker hosted by China Meteorological Administration using the following details: - -- host: gb.wis.cma.cn -- port: 8883 -- username: everyone -- password: everyone - -mqtt-explorer-global-broker-connection - -Click on the 'ADVANCED' button and add the following topics to subscribe to: - -- `origin/#` -- `cache/#` - -mqtt-explorer-global-broker-advanced - -!!! note - When setting up MQTT subscriptions you can use the following wildcards: - - - **Single-level (+)**: a single-level wildcard replaces one topic level - - **Multi-level (#)**: a multi-level wildcard replaces multiple topic levels - -Click 'BACK', then 'SAVE' to save your connection and subscription details. Then click 'CONNECT': - -Messages should start appearing in your MQTT Explorer session as follows: - -mqtt-explorer-global-broker-topics - -You are now ready to start exploring the WIS2 topics and message structure. - -## Exercise 1: Review the WIS2 topic structure - -Use MQTT to browse topic structure under the `origin` and `cache` topics. - -!!! question - - How can we distinguish the originating country providing the data? - -??? success "Click to reveal answer" - - We can distinguish the originating country by looking at the fourth level of the topic structure. For example, the following topic: - - `origin/a/wis2/zmb/zambia_met_service/data/core/weather/surface-based-observations/synop` - - tells us that the data was published by Zambia (zmb). - - The fifth level of the topic structure provides the id of the centre where the data originated from - (in this case, the Zambia Meteorological Service). - -!!! question - - How can we distinguish the data type? - -??? success "Click to reveal answer" - - We can distinguish the data type by looking at the ninth level of the topic structure. For example, the following topic: - - `origin/a/wis2/zmb/zambia_met_service/data/core/weather/surface-based-observations/synop` - - tells us that the data type is surface-based observations (synop). - -## Exercise 2: Review the WIS2 message structure - -Disconnect from MQTT Explorer and update the 'Advanced' sections to change the subscription to the following: - -* `origin/a/wis2/+/+/data/core/weather/surface-based-observations/synop` -* `cache/a/wis2/+/+/data/core/weather/surface-based-observations/synop` - -mqtt-explorer-global-broker-topics-exercise2 - -!!! note - The `+` wildcard is used to subscribe to all countries (fourth level) and centres (fifth level) while the remaining topic structure is fixed to ensure we subscribe to subtopics `data/core/weather/surface-based-observations/synop`. - -Messages should start to appear again. - -You can view the content of the WIS2 message in the "Value" section on the right hand side. - -!!! question - - How can we identify the timestamp that the data was published? And how can we identify the timestamp that the data was collected? - -??? success "Click to reveal answer" - - The timestamp that the data was published is contained in the `properties` section of the message with a key of `pubtime`. - - The timestamp that the data was collected is contained in the `properties` section of the message with a key of `datetime`. - -!!! question - - How can we download the data from the URL provided in the message? - -??? success "Click to reveal answer" - - The URL is contained in the `links` section with `rel="canonical"` and defined by the `href` key. - - You can copy the URL and paste it into a web browser to download the data. - -## Exercise 3: cache vs origin topics - -The same message is published on both the `origin` and `cache` topics. Find a message that has been published on both topics and compare the content. - -!!! question - - What is the difference between the messages published on the `origin` and `cache` topics? - -??? success "Click to reveal answer" - - The message published on the `origin` topic contains a URL to the original data. The message published on the `cache` topic contains a URL to the data cached by the Global Cache. - -## Conclusion - -!!! success "Congratulations!" - In this practical session, you learned: - - - how to subscribe to WIS2 Global Broker services using MQTT Explorer - - the WIS2 topic structure - - the WIS2 notification message structure - - the difference between Global Broker messages published on the `origin` and `cache` topics diff --git a/documentation/docs/practical-sessions/converting-csv-data-to-bufr.md b/documentation/docs/practical-sessions/converting-csv-data-to-bufr.en.md similarity index 85% rename from documentation/docs/practical-sessions/converting-csv-data-to-bufr.md rename to documentation/docs/practical-sessions/converting-csv-data-to-bufr.en.md index d6e928d0..330b6e74 100644 --- a/documentation/docs/practical-sessions/converting-csv-data-to-bufr.md +++ b/documentation/docs/practical-sessions/converting-csv-data-to-bufr.en.md @@ -5,11 +5,12 @@ title: Converting CSV data to BUFR # Converting CSV data to BUFR !!! abstract "Learning outcomes" - By the end of this practical session, you will have: + By the end of this practical session, you will be able to: - - learnt to format CSV data for use with the default automatic weather station BUFR template - - learnt to use **wis2box webapp** to validate and convert sample data - - learnt about some of the common issues that can occur, such as incorrect units + - format CSV data for use with the default automatic weather station BUFR template + - use **wis2box webapp** to validate and convert sample data for AWS stations to BUFR + - use the MinIO UI to upload input CSV data files + - use the dataset editor to use a different template for CSV to BUFR conversion ## Introduction @@ -78,27 +79,21 @@ Inspect the expected columns from the table above and compare to the example dat e.g. ``,"",``. !!! hint "Missing data" - It is recognised that data may be missing for a variety of reasons, whether due to sensor + It is recognized that data may be missing for a variety of reasons, whether due to sensor failure or the parameter not being observed. In these cases missing data can be encoded as per the above answer, the other data in the report remain valid. -!!! bug "Station metadata" - The CSV to BUFR conversion tool has been developed as a standalone tool separate from the wis2box. - As such, all the data to be encoded in BUFR must be present in the CSV file, including the station metadata - like the location and sensor heights. In future versions the web-application will be updated to load the metadata - from the wis2box API and to insert the values into the CSV, similar to the process used in FM-12 SYNOP to BUFR. +### Exercise 2 - converting your first message - Even though all the metadata is contained within the CSV file the station metadata must still be registered in the - wis2box for the message to be procesed. +Now that you have familiarized yourself with the input data file and specified CSV format you will convert this example file to BUFR using the wis2box web-application. +First you need to register the station referred to in the input-data in your wis2box-instance. However, if you try to import *0-20000-0-99100* from OSCAR/Surface using the wis2box web-app you will find that the station does not exist: -### Exercise 2 - converting your first message +
Image showing station not found in OSCAR/Surface
+ +Click the button 'CREATE NEW STATION' and you will be allowed to enter the station details manually. -Now that you have familiarised yourself with the input data file and specified CSV format you will convert this -example file to BUFR using the wis2box web-application. First we need to register the station in the wis2box, in this case the -station is a fictional station located on the island of Sicily in Italy. Navigate to the station list page, -click add station and then search (leave the WIGOS Station Identifier box empty). Click create new station and -enter the fictional station details, suggested details are below: +Enter the fictional station details, suggested details are below: | Field | Value | | ----- |-----------------| @@ -114,8 +109,11 @@ enter the fictional station details, suggested details are below: | Territory or member operating the station | Italy | | Operating status | operational | -Select the appropriate topic, enter the ``collections/stations`` token previously created and click save. You are now -ready to process data from this station. Navigate to CSV to BUFR submission page on the the wis2box web-application +Select the appropriate topic, enter the ``collections/stations`` token previously created and click save. + +You are now ready to process data from this station. + +Navigate to CSV to BUFR submission page on the the wis2box web-application (``http:///wis2box-webapp/csv2bufr_form``). Click the entry box or drag and drop the test file you have downloaded to the entry box. You should now be able to click next to preview and validate the file. @@ -128,9 +126,9 @@ about missing data but in this exercise these can be ignored.
Image showing CSV to BUFR example validation page with warnings
-The next button will take you to the topic selection page, -as with the FM-12 SYNOP to BUFR page select the topic configured for "surface weather observations" on your wis2box -and click next. You should now be on an authorisation page where you will be asked to enter the ``processes/wis2box`` +Click *next* to proceed and you will be asked to provide a dataset-id for the data to be published. Select the dataset-id you create previously and click *next*. + +You should now be on an authorization page where you will be asked to enter the ``processes/wis2box`` token you have previously created. Enter this token and click the "Publish on WIS2" toggle to ensure "Publish to WIS2" is selected (see screenshot below). diff --git a/documentation/docs/practical-sessions/converting-synop-data-to-bufr.md b/documentation/docs/practical-sessions/converting-synop-data-to-bufr-form.en.md similarity index 57% rename from documentation/docs/practical-sessions/converting-synop-data-to-bufr.md rename to documentation/docs/practical-sessions/converting-synop-data-to-bufr-form.en.md index 10cea36e..c221ed9d 100644 --- a/documentation/docs/practical-sessions/converting-synop-data-to-bufr.md +++ b/documentation/docs/practical-sessions/converting-synop-data-to-bufr-form.en.md @@ -2,7 +2,7 @@ title: Converting SYNOP data to BUFR --- -# Converting SYNOP data to BUFR +# Converting SYNOP data to BUFR using the wis2box-webapp !!! abstract "Learning outcomes" By the end of this practical session, you will be able to: @@ -14,104 +14,27 @@ title: Converting SYNOP data to BUFR ## Introduction -Surface weather reports from land surface stations have historically been reported hourly or at the main -(00, 06, 12 and 18 UTC) and intermediate (03, 09, 15, 21 UTC) synoptic hours. Prior to the migration -to BUFR these reports were encoded in the plain text FM-12 SYNOP code form. Whilst the migration to BUFR -was scheduled to be complete by 2012 a large number of reports are still exchanged in the legacy -FM-12 SYNOP format. Further information on the FM-12 SYNOP format can be found in the WMO Manual on Codes, -Volume I.1 (WMO-No. 306, Volume I.1). - -[WMO Manual on Codes, Volume I.1](https://library.wmo.int/records/item/35713-manual-on-codes-international-codes-volume-i-1) - -To aid with completing migration to BUFR some tools have been developed for -encoding FM-12 SYNOP reports to BUFR, in this session you will learn how to use these tools as well -as the relationship between the information contained in the FM-12 SYNOP reports and BUFR messages. +To allow manual observers to submit data directly to the WIS2.0, the wis2box-webapp has a form for converting FM-12 SYNOP bulletins to BUFR. The form also allows users to diagnose and fix simple coding errors in the FM-12 SYNOP bulletin prior to format conversion and exchange and inspect the resulting BUFR data. ## Preparation !!! warning "Prerequisites" - Ensure that your wis2box has been configured and started. - - Confirm the status by visiting the wis2box API (``http:///oapi``) and verifying that the API is running. - - Make sure that you have MQTT Explorer open and connected to your broker. - -In order to submit data to be processed in the wis2box-webapp you will need an auth token for "processes/wis2box". - -You can generate this token by logging in to the wis2box management container and using the `wis2box auth add-token` command: - -```bash -cd ~/wis2box-1.0b5 -python3 wis2box-ctl.py login -wis2box auth add-token --path processes/wis2box -``` - -Or if you want to define your own token: - -```bash -wis2box auth add-token --path processes/wis2box DataIsMagic -``` - -Please record the token that is generated, you will need this later in the session. - -For practical purposes the exercises in this session use data from Romania, import the station ``0-20000-0-15015`` into your station list and associate it with the topic for your "Surface weather observations collection". You can remove this station at the end of the session. - -If you have forgotten your auth token for "collections/stations" you can create a new one with: - -```bash -wis2box auth add-token --path collections/stations -``` - -## Inspecting SYNOP data and BUFR conversion - -### Exercise 1 - the basics - -Review the FM-12 SYNOP message below: + - Open a terminal and connect to your student VM using SSH. + - Connect to the MQTT broker of your wis2box instance using MQTT Explorer. + - Open the wis2box web application (``http:///wis2box-webapp``) and ensure you are logged in. -``` {.copy} -AAXX 21121 -15015 02999 02501 10103 21090 39765 42952 57020 60001= -``` +## Using the wis2box-webapp to convert FM-12 SYNOP to BUFR -Identify the key components of the FM-12 SYNOP message and confirm that the station(s) is (are) registered -within the wis2box. The station(s) should be discoverable via the station list page -(``http:///wis2box-webapp/station``) or via the API directly -(``http:///oapi/collections/stations``). - -!!! hint - The traditional station identifier (2 digit block number and 3 digit station number) are included - in the FM-12 SYNOP report. These can be used to find the station. - -!!! question - What is the traditional station identifier of the station included in the message and - where is the station located? - -??? success "Click to reveal answer" - The five digit group, ``15015``, gives the 5 digit traditional station identifier. In - this case for the station "OCNA SUGATAG" located in Romania. +### Exercise 1 - using the wis2box-webapp to convert FM-12 SYNOP to BUFR +Make sure you have the auth token for "processes/wis2box" that you generated in the previous exercise and that you are connected to your wis2box broker in MQTT Explorer. -Identify the number of weather reports in the message. - -!!! question - How many weather reports are in the message? - -??? success "Click to reveal answer" - One, the report contains a single message. - - The first line ``AAXX 21121`` indicates that this is an FM-12 SYNOP message (``AAXX``), - the 2112 indicates that the weather observation was made on the 21st day of the month at 12 UTC. - The final digit of the row, ``1``, indicates the source and units of the wind speed. The second line - contains a single weather report, beginning with the 5 digit group ``15015`` and ending with - the ``=`` symbol. - -### Exercise 2 - converting your first message - -Now that you have reviewed the message, you are ready to convert the data to BUFR. - -Copy the message you have just reviewed: +Copy the following message: ``` {.copy} -AAXX 21121 +AAXX 27031 15015 02999 02501 10103 21090 39765 42952 57020 60001= ``` @@ -119,28 +42,38 @@ Open the wis2box web application and navigate to the synop2bufr page using the l - Paste the content you have copied in the text entry box. - Select the month and year using the date picker, assume the current month for this exercise. -- Select a topic from the drop down menu (the options are based on the metadata configured in the wis2box) +- Select a topic from the drop down menu (the options are based on the datasets configured in the wis2box). - Enter the "processes/wis2box" auth token you generated earlier - Ensure "Publish on WIS2" is toggled ON - Click "SUBMIT"
Dialog showing synop2bufr page, including toggle button
-Click submit. The data will now be converted to BUFR and the result returned to the web application. +Click submit. You will receive an warning message as the station is not registered in the wis2box. Go to the station-editor and import the following station: + +``` {.copy} +0-20000-0-15015 +``` + +Ensure the station is associated with the topic you selected in the previous step and then return to the synop2bufr page and repeat the process with the same data as before. -Click on "Output BUFR files" to see a list of the files that have been generated. +!!! question + How can you see the result of the conversion from FM-12 SYNOP to BUFR? + +??? success "Click to reveal answer" + The result section of the page shows Warnings, Errors and Output BUFR files. + + Click on "Output BUFR files" to see a list of the files that have been generated. You should see one file listed. + + The download button allows the BUFR data to be downloaded directly to your computer. -??? tip - The result section of the page should show a single converted BUFR message with zero warnings - or errors. The download button allows the BUFR data to be downloaded directly to your computer. - Click the down arrow / chevron to reveal download and inspect buttons. The inspect button runs a process to convert and extract the data from BUFR.
Dialog showing result of successfully submitting a message
!!! question - The FM-12 SYNOP format does not include the station location, elevation or barometer height. + The FM-12 SYNOP input data did not include the station location, elevation or barometer height. Confirm that these are in the output BUFR data, where do these come from? ??? success "Click to reveal answer" @@ -155,20 +88,14 @@ Click on "Output BUFR files" to see a list of the files that have been generated The BUFR file can also be inspected by downloading the file and validating using a tool such as as the ECMWF ecCodes BUFR validator. -As a final step navigate to the monitoring page from the left menu and confirm that the data have been published. - -
Image showing monitoring tab in on the left menu
- -
Image showing monitoring page and published data
+Go to MQTT Explorer and check the WIS2 notifications topic to see the WIS2 notifications that have been published. -You should also be able to see these notifications in MQTT Explorer. - -### Exercise 3 - understanding the station list +### Exercise 2 - understanding the station list For this next exercise you will convert a file containing multiple reports, see the data below: ``` {.copy} -AAXX 21121 +AAXX 27031 15015 02999 02501 10103 21090 39765 42952 57020 60001= 15020 02997 23104 10130 21075 30177 40377 58020 60001 81041= 15090 02997 53102 10139 21075 30271 40364 58031 60001 82046= @@ -199,15 +126,21 @@ AAXX 21121 identifier is given by the traditional station identifier with ``0-20000-0`` prepended, e.g. ``15015`` has become ``0-20000-0-15015``. -Using the station list page from the web application import the missing stations from [OSCAR/Surface](https://oscar.wmo.int/surface/) -into the wis2box and repeat the exercise. +Using the station list page, import the following stations: + +``` {.copy} +0-20000-0-15020 +0-20000-0-15090 +``` + +Ensure that the stations are associated with the topic you selected in the previous exercise and then return to the synop2bufr page and repeat the process. -Three BUFR files should be generated and there should be no warnings or errors listed in the web application. +Three BUFR files should now be generated and there should be no warnings or errors listed in the web application. In addition to the basic station information, additional metadata such as the station elevation above sea level and the barometer height above sea level are required for encoding to BUFR. The fields are included in the station list and station editor pages. -### Exercise 4 - debugging +### Exercise 3 - debugging In this final exercise you will identify and correct two of the most common problems encountered when using this tool to convert FM-12 SYNOP to BUFR. @@ -221,7 +154,7 @@ may be prior to submitting the data through the web application. has been clicked. ``` {.copy} -AAXX 21121 +AAXX 27031 15015 02999 02501 10103 21090 39765 42952 57020 60001 15020 02997 23104 10130 21075 30177 40377 58020 60001 81041= 15090 02997 53102 10139 21075 30271 40364 58031 60001 82046= @@ -240,7 +173,7 @@ The second example below contains several common issues found in FM-12 SYNOP rep data and try to identify the issues and then submit the corrected data through the web application. ```{.copy} -AAXX 21121 +AAXX 27031 15020 02997 23104 10/30 21075 30177 40377 580200 60001 81041= ``` @@ -267,14 +200,13 @@ the stations removed from the list after deleting.
Station metadata viewer
-You can also delete the file used in the final exercise as this will no longer be required. - ## Conclusion !!! success "Congratulations!" In this practical session, you learned: + - how the synop2bufr tool can be used to convert FM-12 SYNOP reports to BUFR; - how to submit a FM-12 SYNOP report through the web-app; - how to diagnose and correct simple errors in an FM-12 SYNOP report; - the importance of registering stations in the wis2box (and OSCAR/Surface); diff --git a/documentation/docs/practical-sessions/converting-synop-data-to-bufr.en.md b/documentation/docs/practical-sessions/converting-synop-data-to-bufr.en.md new file mode 100644 index 00000000..ed6d9d5b --- /dev/null +++ b/documentation/docs/practical-sessions/converting-synop-data-to-bufr.en.md @@ -0,0 +1,397 @@ +--- +title: Converting SYNOP data to BUFR +--- + +# Converting SYNOP data to BUFR from the command line + +!!! abstract "Learning outcomes" + By the end of this practical session, you will be able to: + + - use the synop2bufr tool to convert FM-12 SYNOP reports to BUFR; + - diagnose and fix simple coding errors in FM-12 SYNOP reports prior to format conversion; + +## Introduction + +Surface weather reports from land surface stations have historically been reported hourly or at the main +(00, 06, 12 and 18 UTC) and intermediate (03, 09, 15, 21 UTC) synoptic hours. Prior to the migration +to BUFR these reports were encoded in the plain text FM-12 SYNOP code form. Whilst the migration to BUFR +was scheduled to be complete by 2012 a large number of reports are still exchanged in the legacy +FM-12 SYNOP format. Further information on the FM-12 SYNOP format can be found in the WMO Manual on Codes, +Volume I.1 (WMO-No. 306, Volume I.1). + +[WMO Manual on Codes, Volume I.1](https://library.wmo.int/records/item/35713-manual-on-codes-international-codes-volume-i-1) + +To aid with completing migration to BUFR some tools have been developed for +encoding FM-12 SYNOP reports to BUFR, in this session you will learn how to use these tools as well +as the relationship between the information contained in the FM-12 SYNOP reports and BUFR messages. + +## Preparation + +!!! warning "Prerequisites" + + - Ensure that your wis2box has been configured and started. + - Confirm the status by visiting the wis2box API (``http:///oapi``) and verifying that the API is running. + - Make sure to read the **synop2bufr primer** and **ecCodes primer** sections before starting the exercises. + +## synop2bufr primer + +Below are essential `synop2bufr` commands and configurations: + +### transform +The `transform` function converts a SYNOP message to BUFR: + +```bash +synop2bufr data transform --metadata my_file.csv --output-dir ./my_directory --year message_year --month message_month my_SYNOP.txt +``` + +Note that if the metadata, output directory, year and month options are not specified, they will assume their default values: + +| Option | Default | +| ----------- | ----------- | +| --metadata | station_list.csv | +| --output-dir | The current working directory. | +| --year | The current year. | +| --month | The current month. | + +!!! note + One must be cautious using the default year and month, as the day of the month specified in the report may not correspond (e.g. June does not have 31 days). + +In the examples, the year and month are not given, so feel free to specify a date yourself or use the default values. + +## ecCodes primer + +ecCodes provides both command line tools and can be embedded in your own applications. Below are some useful command +line utilities to work with BUFR data. + +### bufr_dump + +The `bufr_dump` command is a generic BUFR information tool. It has many options, but the following will be the most applicable to the exercises: + +```bash +bufr_dump -p my_bufr.bufr4 +``` + +This will display BUFR content to your screen. If you are interested in the values taken by a variable in particular, use the `egrep` command: + +```bash +bufr_dump -p my_bufr.bufr4 | egrep -i temperature +``` + +This will display variables related to temperature in your BUFR data. If you want to do this for multiple types of variables, filter the output using a pipe (`|`): + +```bash +bufr_dump -p my_bufr.bufr4 | egrep -i 'temperature|wind' +``` + +## Converting FM-12 SYNOP to BUFR using synop2bufr from the command line + +The eccodes library and synop2bufr module are installed in the wis2box-api container. In order to do the next few exercises we will copy the synop2bufr-exercises directory to the wis2box-api container and run the exercises from there. + +```bash +docker cp ~/exercise-materials/synop2bufr-exercises wis2box-api:/root +``` + +Now we can enter the container and run the exercises: + +```bash +docker exec -it wis2box-api /bin/bash +``` + +### Exercise 1 +Navigate to the `/root/synop2bufr-exercises/ex_1` directory and inspect the SYNOP message file message.txt: + +```bash +cd /root/synop2bufr-exercises/ex_1 +more message.txt +``` + +!!! question + + How many SYNOP reports are in this file? + +??? success "Click to reveal answer" + + There is 1 SYNOP report, as there is only 1 delimiter (=) at the end of the message. + +Inspect the station list: + +```bash +more station_list.csv +``` + +!!! question + + How many stations are listed in the station list? + +??? success "Click to reveal answer" + + There is 1 station, the station_list.csv contains one row of station metadata. + +!!! question + Try to convert `message.txt` to BUFR format. + +??? success "Click to reveal answer" + + To convert the SYNOP message to BUFR format, use the following command: + + ```bash + synop2bufr data transform --metadata station_list.csv --output-dir ./ --year 2024 --month 09 message.txt + ``` + +!!! tip + + See the [synop2bufr primer](#synop2bufr-primer) section. + +Inspect the resulting BUFR data using `bufr_dump`. + +!!! question + Find how to compare the latitude and longitude values to those in the station list. + +??? success "Click to reveal answer" + + To compare the latitude and longitude values in the BUFR data to those in the station list, use the following command: + + ```bash + bufr_dump -p WIGOS_0-20000-0-15015_20240921T120000.bufr4 | egrep -i 'latitude|longitude' + ``` + + This will display the latitude and longitude values in the BUFR data. + +!!! tip + + See the [ecCodes primer](#eccodes-primer) section. + +### Exercise 2 +Navigate to the `exercise-materials/synop2bufr-exercises/ex_2` directory and inspect the SYNOP message file message.txt: + +```bash +cd /root/synop2bufr-exercises/ex_2 +more message.txt +``` + +!!! question + + How many SYNOP reports are in this file? + +??? success "Click to reveal answer" + + There are 3 SYNOP reports, as there are 3 delimiters (=) at the end of the message. + +Inspect the station list: + +```bash +more station_list.csv +``` + +!!! question + + How many stations are listed in the station list? + +??? success "Click to reveal answer" + + There are 3 stations, the station_list.csv contains three rows of station metadata. + +!!! question + Convert `message.txt` to BUFR format. + +??? success "Click to reveal answer" + + To convert the SYNOP message to BUFR format, use the following command: + + ```bash + synop2bufr data transform --metadata station_list.csv --output-dir ./ --year 2024 --month 09 message.txt + ``` + +!!! question + + Based on the results of the exercises in this and the previous exercise, how would you predict the number of + resulting BUFR files based upon the number of SYNOP reports and stations listed in the station metadata file? + +??? success "Click to reveal answer" + + To see the produced BUFR-files run the following command: + + ```bash + ls -l *.bufr4 + ``` + + The number of BUFR files produced will be equal to the number of SYNOP reports in the message file. + +Inspect the resulting BUFR data using `bufr_dump`. + +!!! question + How can you check the WIGOS Station ID encoded inside the BUFR data of each file produced? + +??? success "Click to reveal answer" + + This can be done using the following commands: + + ```bash + bufr_dump -p WIGOS_0-20000-0-15015_20240921T120000.bufr4 | egrep -i 'wigos' + ``` + + ```bash + bufr_dump -p WIGOS_0-20000-0-15020_20240921T120000.bufr4 | egrep -i 'wigos' + ``` + + ```bash + bufr_dump -p WIGOS_0-20000-0-15090_20240921T120000.bufr4 | egrep -i 'wigos' + ``` + + Note that if you have a directory with just these 3 BUFR files, you can use Linux wildcards as follows: + + ```bash + bufr_dump -p *.bufr4 | egrep -i 'wigos' + ``` + +### Exercise 3 +Navigate to the `exercise-materials/synop2bufr-exercises/ex_3` directory and inspect the SYNOP message file message.txt: + +```bash +cd /root/synop2bufr-exercises/ex_3 +more message.txt +``` + +This SYNOP message only contains one longer report with more sections. + +Inspect the station list: + +```bash +more station_list.csv +``` + +!!! question + + Is it problematic that this file contains more stations than there are reports in the SYNOP message? + +??? success "Click to reveal answer" + + No, this is not a problem provided that there exists a row in the station list file with a station TSI matching that of the SYNOP report we are trying to convert. + +!!! note + + The station list file is a source of metadata for `synop2bufr` to provide the information missing in the alphanumeric SYNOP report and required in the BUFR SYNOP. + +!!! question + Convert `message.txt` to BUFR format. + +??? success "Click to reveal answer" + + This is done using the `transform` command, for example: + + ```bash + synop2bufr data transform --metadata station_list.csv --output-dir ./ --year 2024 --month 09 message.txt + ``` + +Inspect the resulting BUFR data using `bufr_dump`. + +!!! question + + Find the following variables: + + - Air temperature (K) of the report + - Total cloud cover (%) of the report + - Total period of sunshine (mins) of the report + - Wind speed (m/s) of the report + +??? success "Click to reveal answer" + + To find the variables by keyword in the BUFR data, you can use the following commands: + + ```bash + bufr_dump -p WIGOS_0-20000-0-15260_20240921T115500.bufr4 | egrep -i 'temperature' + ``` + + You can use the following command to search for multiple keywords: + + ```bash + bufr_dump -p WIGOS_0-20000-0-15260_20240921T115500.bufr4 | egrep -i 'temperature|cover|sunshine|wind' + ``` + +!!! tip + + You may find the last command of the [ecCodes primer](#eccodes-primer) section useful. + + +### Exercise 4 +Navigate to the `exercise-materials/synop2bufr-exercises/ex_4` directory and inspect the SYNOP message file message.txt: + +```bash +cd /root/synop2bufr-exercises/ex_4 +more message_incorrect.txt +``` + +!!! question + + What is incorrect about this SYNOP file? + +??? success "Click to reveal answer" + + The SYNOP report for 15015 is missing the delimiter (`=`) that allows `synop2bufr` to distinguish this report from the next. + +Attempt to convert `message_incorrect.txt` using `station_list.csv` + +!!! question + + What problem(s) did you encounter with this conversion? + +??? success "Click to reveal answer" + + To convert the SYNOP message to BUFR format, use the following command: + + ```bash + synop2bufr data transform --metadata station_list.csv --output-dir ./ --year 2024 --month 09 message_incorrect.txt + ``` + + Attempting to convert should raise the following errors: + + - `[ERROR] Unable to decode the SYNOP message` + - `[ERROR] Error parsing SYNOP report: AAXX 21121 15015 02999 02501 10103 21090 39765 42952 57020 60001 15020 02997 23104 10130 21075 30177 40377 58020 60001 81041. 10130 is not a valid group!` + +### Exercise 5 +Navigate to the `exercise-materials/synop2bufr-exercises/ex_5` directory and inspect the SYNOP message file message.txt: + +```bash +cd /root/synop2bufr-exercises/ex_5 +more message.txt +``` + +Attempt to convert `message.txt` to BUFR format using `station_list_incorrect.csv` + +!!! question + + What problem(s) did you encounter with this conversion? + Considering the error presented, justify the number of BUFR files produced. + +??? success "Click to reveal answer" + + To convert the SYNOP message to BUFR format, use the following command: + + ```bash + synop2bufr data transform --metadata station_list_incorrect.csv --output-dir ./ --year 2024 --month 09 message.txt + ``` + + One of the station TSIs (`15015`) has no corresponding metadata in the station-list, which will prohibit synop2bufr from accessing additional necessary metadata to convert the first SYNOP report to BUFR. + + You will see the following warning: + + - `[WARNING] Station 15015 not found in station file` + + You can see the number of BUFR files produced by running the following command: + + ```bash + ls -l *.bufr4 + ``` + + There are 3 SYNOP reports in message.txt but only 2 BUFR files have been produced. This is because one of the SYNOP reports lacked the necessary metadata as mentioned above. + +## Conclusion + +!!! success "Congratulations!" + + In this practical session, you learned: + + - how the synop2bufr tool can be used to convert FM-12 SYNOP reports to BUFR; + - how to diagnose and fix simple coding errors in FM-12 SYNOP reports prior to format conversion; + + diff --git a/documentation/docs/practical-sessions/datasets-with-access-control.md b/documentation/docs/practical-sessions/datasets-with-access-control.md new file mode 100644 index 00000000..a39d1830 --- /dev/null +++ b/documentation/docs/practical-sessions/datasets-with-access-control.md @@ -0,0 +1,122 @@ +--- +title: Setting up a recommended dataset with access control +--- + +# Setting up a recommended dataset with access control + +!!! abstract "Learning outcomes" + By the end of this practical session, you will be able to: + + - create a new dataset with data policy 'recommended' + - add an access token to the dataset + - validate the dataset can not be accessed without the access token + - add the access token to HTTP headers to access the dataset + +## Introduction + +Datasets that are not considered 'core' dataset in WMO can optionally be configured with an access control policy. wis2box provides a mechanism to add an access token to a dataset which will prevent users from downloading data unless they supply the access token in the HTTP headers. + +## Preparation + +Ensure you have SSH access to your student VM and that your wis2box instance is up and running. + +Make sure you are connected to the MQTT broker of your wis2box instance using MQTT Explorer. You can use the public credentials `everyone/everyone` to connect to the broker. + +Ensure you have a web browser open with the wis2box-webapp for your instance by going to `http:///wis2box-webapp`. + +## create a new dataset with data policy 'recommended' + +Go to the 'dataset editor' page in the wis2box-webapp and create a new dataset. Use the same centre-id as in the previous practical sessions and use the template='surface-weather-observations/synop'. + +You may get a pop-up message that there already is a dataset with the same metadata identifier: + +provide-a-new-dataset-id + +Click 'OK' to proceed. + +In the dataset editor, set the data policy to 'recommended' (note that this will update the identifier and replace 'core' with 'reco') and fill all the required fields. + +Ensure the dataset is published and the WIS2 Notification Message announcing the new Discovery Metadata record is published. + +Add the following stations to your "recommended" dataset to ensure the test data can be ingested and published: + +- 0-20000-0-60351 +- 0-20000-0-60355 +- 0-20000-0-60360 + +## add an access token to the dataset + +Login to the wis2box-management container, + +```bash +docker exec -it wis2box-management bash +``` + +From command line inside the container you can secure a dataset using the `wis2box auth add-token` command, using the flag `--mdi` to specify the metadata-identifier of the dataset and the access token as an argument. + +For example, to add the access token `S3cr3tT0k3n` to the dataset with metadata-identifier `urn:md:wmo:mydataset`: + +```bash +wis2box auth add-token --mdi urn:md:wmo:mydataset S3cr3tT0k3n +``` + +## publish some data to the dataset + +Copy the file `exercise-materials/access-control-exercises/aws-example2.csv` to the directory defined by `WIS2BOX_HOST_DATADIR` in your `wis2box.env`: + +```bash +cp ~/exercise-materials/access-control-exercises/aws-example2.csv ~/wis2box-data +``` + +Then login to the **wis2box-management** container: + +```bash +cd ~/wis2box-1.0b8 +python3 wis2box-ctl.py login +``` + +From the wis2box command line we can ingest the sample data file `aws-example2.csv` into a specific dataset as follows: + +```bash +wis2box data ingest -p /data/wis2box/aws-example2.csv--metadata-id urn:md:wmo:mydataset +``` + +Make sure to provide the correct metadata-identifier for your dataset and check that you receive WIS2 data-notifications in MQTT Explorer. + +Check the canonical link in the WIS2 Notification Message and copy/paste the link to the browser to try and download the data. + +You should see a 403 Forbidden error. + +## add the access token to HTTP headers to access the dataset + +In order to demonstrate that the access token is required to access the dataset we will reproduce the error you saw in the browser using the command line function `wget`. + +From the command line in your student VM, use the `wget` command with the canonical-link you copied from the WIS2 Notification Message. + +```bash +wget +``` + +You should see that the HTTP request returns with *401 Unauthorized* and the data is not downloaded. + +Now add the access token to the HTTP headers to access the dataset. + +```bash +wget --header="Authorization: Bearer S3cr3tT0k3n" +``` + +Now the data should be downloaded successfully. + +## clean up + +Delete the dataset using the dataset editor. + +## Conclusion + +!!! success "Congratulations!" + In this practical session, you learned how to: + + - create a new dataset with data policy 'recommended' + - add an access token to the dataset + - validate the dataset can not be accessed without the access token + - add the access token to HTTP headers to access the dataset diff --git a/documentation/docs/practical-sessions/downloading-and-finding-data-from-wis2.md b/documentation/docs/practical-sessions/downloading-and-finding-data-from-wis2.md deleted file mode 100644 index 7d6e862a..00000000 --- a/documentation/docs/practical-sessions/downloading-and-finding-data-from-wis2.md +++ /dev/null @@ -1,183 +0,0 @@ ---- -title: Downloading and finding data from WIS2 ---- - -# Downloading and finding data from WIS2 - -!!! abstract "Learning outcomes!" - - By the end of this practical session, you will be able to: - - - use pywis-pubsub to subscribe to a Global Broker and download data to your local system - - use pywiscat to discover datasets from the Global Discovery Catalogue - -## Introduction - -In this session you will learn how to discover data from the WIS2 Global Discovery Catalogue (GDC) and download data from a WIS2 Global Broker (GB). - -## Preparation - -!!! note - Before starting please login to your student VM. - -## Downloading data with pywis-pubsub - -The [first practical session](../connecting-to-mqtt) used MQTT Explorer to connect to the Météo-France Global Broker. - -Let's use the [pywis-pubsub](https://github.com/wmo-im/pywis-pubsub) to subscribe using a command line tool. - -```bash -pywis-pubsub subscribe --help -``` - -!!! note - pywis-pubsub is pre-installed on the local training environment, but can be installed from anywhere with `pip3 install pywis-pubsub` - -Update the sample configuration (see the sections marked **TBD**) to connect to the Météo-France Global Broker: - -```bash -vi ~/exercise-materials/pywis-pubsub-exercises/config.yml -``` - -Update the following values in the configuration: - -- **broker**: `mqtts://everyone:everyone@globalbroker.meteo.fr:8883` -- **subscribe_topics**: fill in one to many topics `cache/#` (on separate lines) -- **storage.option.path**: add a directory to where data should be downloaded - -Run the `pywis-pubsub` command: - -```bash -pywis-pubsub subscribe -c ~/exercise-materials/pywis-pubsub-exercises/config.yml --verbosity DEBUG -``` - -At this point you should see a number of ongoing messages on your screen. Hit `Crtl-C` to stop messages from arriving/download to help analyze the content. - -!!! note - - The above command will download data to your local system for demo purposes. For operational environments - you will need to consider and manage diskspace requirements as part of your workflow. - -!!! question - - What is the format of the data notifications that are displayed on the screen? - -??? success "Click to reveal answer" - The format is GeoJSON - -!!! question - - Is there data being downloaded? How can we run the `pywis-pubsub` command to be able to download the data (hint: review the options when running the `pywis-pubsub subscribe --help` command)? - -??? success "Click to reveal answer" - Add the `-d` or `--download` flag to the `pywis-pubsub` command - -Stop the `pywis-pubsub` command (CTRL-C) and update the configuration to be able to download the data -to `/tmp/wis2-data`. - -Try spatial filtering with a bounding box: - -```bash -pywis-pubsub subscribe -c ~/exercise-materials/pywis-pubsub-exercises/config.yml --verbosity INFO -d -b -142,42,-52,84 -``` - -!!! note - - Try using your own bounding box (format is `west,south,east,north`, in decimal degrees). - -## Finding data with pywiscat - -Let's use [pywiscat](https://github.com/wmo-im/pywiscat) to query the GDC - -At the moment, the available GDCs are as follows: - -- Environment and Climate Change Canada, Meteorological Service of Canada: https://api.weather.gc.ca/collections/wis2-discovery-metadata -- China Meteorological Administration: https://api.weather.gc.ca/api/collections/wis2-discovery-metadata - -```bash -pywiscat search --help -``` - -!!! note - pywiscat is pre-installed on the local training environment, but can be installed from anywhere with `pip3 install pywiscat` - -!!! note - by default, pywiscat connects to Canada's Global Discovery Catalogue. To set to a different catalogue, set the `PYWISCAT_GDC_URL` - environment variable - -```bash -pywiscat search -``` - -!!! question - - How many records are returned from the search? - -??? success "Click to reveal answer" - There should be 40 records returned - - ``` - Querying WIS2 GDC 🗃️ ... - - Results: 40 records - ... - ``` - -Let's try querying the GDC with a keyword: - -```bash -pywiscat search -q observations -``` - -!!! question - - What is the data policy of the results? - -??? success "Click to reveal answer" - All data returned should specify "core" data - - ``` - +---------------------------------------------------------------------------+------------------------------+-------------------------------------------------------+-------------+ - | id | centre | title | data policy | - +---------------------------------------------------------------------------+------------------------------+-------------------------------------------------------+-------------+ - | urn:x-wmo:md:dma:dominica_met_wis2node:surface-weather-observations | dominica_met_wis2node | Surface weather observations from Dominica Meteoro... | core | - | urn:x-wmo:md:gin:conakry_met_centre:surface-weather-observations | conakry_met_centre | Surface weather observations from gin.conakry_met_... | core | - | urn:x-wmo:md:atg:antigua_met_wis2node:surface-weather-observations | antigua_met_wis2node | Surface weather observations from Antigua and Barb... | core | - | urn:x-wmo:md:bfa:ouagadougou_met_centre:surface-weather-observations | ouagadougou_met_centre | Surface weather observations from bfa.ouagadougou_... | core | - ... - ``` - -Try additional queries with `-q` - -!!! tip - - The `-q` flag allows for the following syntax: - - - `-q synop`: find all records with the word "synop" - - `-q temp`: find all records with the word "temp" - - `-q "observations AND malawi"`: find all records with the words "observations" and "malawi" - - `-q "observations NOT malawi"`: find all records that contain the word "observations" but not the word "malawi" - - `-q "synop OR temp"`: find all records with both "synop" or "temp" - - `-q "obs~"`: fuzzy search - - When searching for terms with spaces, enclose in double quotes. - -Let's get more details on a specific search result that we are interested in: - -```bash -pywiscat get -``` - -!!! tip - - Use the `id` value from the previous search. - - -## Conclusion - -!!! success "Congratulations!" - - In this practical session, you learned how to: - - - use pywis-pubsub to subscribe to a Global Broker and download data to your local system - - use pywiscat to discover datasets from the Global Discovery Catalogue diff --git a/documentation/docs/practical-sessions/downloading-data-from-wis2.en.md b/documentation/docs/practical-sessions/downloading-data-from-wis2.en.md new file mode 100644 index 00000000..5ec45635 --- /dev/null +++ b/documentation/docs/practical-sessions/downloading-data-from-wis2.en.md @@ -0,0 +1,152 @@ +--- +title: Downloading data from WIS2 notifications +--- + +# Downloading data from WIS2 notifications + +!!! abstract "Learning outcomes!" + + By the end of this practical session, you will be able to: + + - use the "wis2downloader" to subscribe to WIS2 data notifications and download data to your local system. + +## Introduction + +In this session you will learn how to setup a subscription to a WIS2 Broker and automatically download data to your local system using the "wis2downloader"-service included in the wis2box. + +!!! note "About wis2downloader" + + The wis2downloader is also available as a standalone service that can be run on a different system from the one that is publishing the WIS2 notifications. See [wis2downloader](https://pypi.org/project/wis2downloader/) for more information for using the wis2downloader as a standalone service. + + If you like to develop your own service for subscribing to WIS2 notifications and downloading data, you can use the [wis2downloader source code](https://github.com/wmo-im/wis2downloader) as a reference. + +## Preparation + +Before starting please login to your student VM and ensure your wis2box instance is up and running. + +## Exercise 1: viewing the wis2download dashboard in Grafana + +Open a web browser and navigate to the Grafana dashboard for your wis2box instance by going to `http://:3000`. + +Click on dashboards in the left-hand menu, and then select the **wis2downloader dashboard**. + +You should see the following dashboard: + +![wis2downloader dashboard](../assets/img/wis2downloader-dashboard.png) + +This dashboard is based on metrics published by the wis2downloader service and will show you the status of the downloads that are currently in progress. + +On the top left corner you can see the subscriptions that are currently active. + +Keep this dashboard open as you will use it to monitor the download progress in the next exercise. + +## Exercise 2: reviewing the wis2downloader configuration + +The wis2downloader-service started by the wis2box-stack can be configured using the environment variables defined in your wis2box.env file. + +The following environment variables are used by the wis2downloader: + + - DOWNLOAD_BROKER_HOST: The hostname of the MQTT broker to connect to. Defaults to globalbroker.meteo.fr + - DOWNLOAD_BROKER_PORT: The port of the MQTT broker to connect to. Defaults to 443 (HTTPS for websockets) + - DOWNLOAD_BROKER_USERNAME: The username to use to connect to the MQTT broker. Defaults to everyone + - DOWNLOAD_BROKER_PASSWORD: The password to use to connect to the MQTT broker. Defaults to everyone + - DOWNLOAD_BROKER_TRANSPORT: websockets or tcp, the transport-mechanism to use to connect to the MQTT broker. Defaults to websockets, + - DOWNLOAD_RETENTION_PERIOD_HOURS: The retention period in hours for the downloaded data. Defaults to 24 + - DOWNLOAD_WORKERS: The number of download workers to use. Defaults to 8. Determines the number of parallel downloads. + - DOWNLOAD_MIN_FREE_SPACE_GB: The minimum free space in GB to keep on the volume hosting the downloads. Defaults to 1. + +To review the current configuration of the wis2downloader, you can use the following command: + +```bash +cat ~/wis2box-1.0b8/wis2box.env | grep DOWNLOAD +``` + +!!! question "Review the configuration of the wis2downloader" + + What is the default MQTT broker that the wis2downloader connects to? + + What is the default retention period for the downloaded data? + +??? success "Click to reveal answer" + + The default MQTT broker that the wis2downloader connects to is `globalbroker.meteo.fr`. + + The default retention period for the downloaded data is 24 hours. + +!!! note "Updating the configuration of the wis2downloader" + + To update the configuration of the wis2downloader, you can edit the wis2box.env file. To apply the changes you can re-run the start command for the wis2box-stack: + + ```bash + python3 wis2box-ctl.py start + ``` + + And you will see the wis2downloader service restart with the new configuration. + +You can keep the default configuration for the purpose of this exercise. + +## Exercise 3: adding subscriptions to the wis2downloader + +Inside the **wis2downloader** container, you can use the command line to list, add and delete subscriptions. + +To login to the **wis2downloader** container, use the following command: + +```bash +python3 wis2box-ctl.py login wis2downloader +``` + +Then use the following command to list the subscriptions that are currently active: + +```bash +wis2downloader list-subscriptions +``` + +This command returns an empty list since no subscriptions are currently active. + +For the purpose of this exercise, we will subscribe to the following topic `cache/a/wis2/de-dwd-gts-to-wis2/#`, to subscribe to data published by the DWD-hosted GTS-to-WIS2 gateway and downloading notifications from the Global Cache. + +To add this subscription, use the following command: + +```bash +wis2downloader add-subscription --topic cache/a/wis2/de-dwd-gts-to-wis2/# +``` + +Then exit the **wis2downloader** container by typing `exit`. + +Check the wis2downloader dashboard in Grafana to see the new subscription added. Wait a few minutes and you should see the first downloads starting. Go to he next exercise once you have confirmed that the downloads are starting. + +## Exercise 4: viewing the downloaded data + +The wis2downloader-service in the wis2box-stack downloads the data in the 'downloads' directory in the directory you defined as the WIS2BOX_HOST_DATADIR in your wis2box.env file. To view the contents of the downloads directory, you can use the following command: + +```bash +ls -R ~/wis2box-data/downloads +``` + +Note that the downloaded data is stored in directories named after the topic the WIS2 Notification was published on. + +## Exercise 5: removing subscriptions from the wis2downloader + +Next, the remove the subscription you made from the wis2downloader, using the following command: + +```bash +wis2downloader remove-subscription --topic cache/a/wis2/de-dwd-gts-to-wis2/# +``` + +Check the wis2downloader dashboard in Grafana to see the subscription removed. You should see the downloads stopping. + +!!! More command line tools + + The following tools can also be used to discover and access data from WIS2: + + - [pywiscat](https://github.com/wmo-im/pywiscat) provides search capability atop the WIS2 Global Discovery Catalogue in support of reporting and analysis of the WIS2 Catalogue and its associated discovery metadata + - [pywis-pubsub](https://github.com/wmo-im/pywis-pubsub) provides subscription and download capability of WMO data from WIS2 infrastructure services + + +## Conclusion + +!!! success "Congratulations!" + + In this practical session, you learned how to: + + - use the 'wis2downloader' to subscribe to a WIS2 Broker and download data to your local system diff --git a/documentation/docs/practical-sessions/image.png b/documentation/docs/practical-sessions/image.png new file mode 100644 index 00000000..e3e68c07 Binary files /dev/null and b/documentation/docs/practical-sessions/image.png differ diff --git a/documentation/docs/practical-sessions/initializing-wis2box.en.md b/documentation/docs/practical-sessions/initializing-wis2box.en.md new file mode 100644 index 00000000..cf7a2762 --- /dev/null +++ b/documentation/docs/practical-sessions/initializing-wis2box.en.md @@ -0,0 +1,436 @@ +--- +title: Initializing wis2box +--- + +# Initializing wis2box + +!!! abstract "Learning outcomes" + + By the end of this practical session, you will be able to: + + - run the `wis2box-create-config.py` script to create the initial configuration + - start wis2box and check the status of its components + - access the **wis2box-webapp**, API, MinIO UI and Grafana dashboard in a browser + - connect to the local **wis2box-broker** using MQTT Explorer + +!!! note + + The current training materials are using wis2box-1.0b8. + + See [accessing-your-student-vm](accessing-your-student-vm.md) for instructions on how to download and install the wis2box software stack if you are running this training outside of a local training session. + +## Preparation + +Login to your designated VM with your username and password and ensure you are in the `wis2box-1.0b8` directory: + +```bash +cd ~/wis2box-1.0b8 +``` + +## Creating the initial configuration + +The initial configuration for the wis2box requires: + +- an environment file `wis2box.env` containing the configuration parameters +- a directory on the host-machine to share between the host machine and the wis2box containers defined by the `WIS2BOX_HOST_DATADIR` environment variable + +The `wis2box-create-config.py` script can be used to create the initial configuration of your wis2box. + +It will ask you a set of question to help setup your configuration. + +You will be able to review and update the configuration files after the script has completed. + +Run the script as follows: + +```bash +python3 wis2box-create-config.py +``` + +### wis2box-host-data directory + +The script will ask you to enter the directory to be used for the `WIS2BOX_HOST_DATADIR` environment variable. + +Note that you need to define the full path to this directory. + +For example if your username is `username`, the full path to the directory is `/home/username/wis2box-data`: + +```{.copy} +username@student-vm-username:~/wis2box-1.0b8$ python3 wis2box-create-config.py +Please enter the directory to be used for WIS2BOX_HOST_DATADIR: +/home/username/wis2box-data +The directory to be used for WIS2BOX_HOST_DATADIR will be set to: + /home/username/wis2box-data +Is this correct? (y/n/exit) +y +The directory /home/username/wis2box-data has been created. +``` + +### wis2box URL + +Next, you will be asked to enter the URL for your wis2box. This is the URL that will be used to access the wis2box web application, API and UI. + +Please use `http://` as the URL. Remember that your hostname is defined by your `username.wis2.training` + +```{.copy} +Please enter the URL of the wis2box: + For local testing the URL is http://localhost + To enable remote access, the URL should point to the public IP address or domain name of the server hosting the wis2box. +http://username.wis2.training +The URL of the wis2box will be set to: + http://username.wis2.training +Is this correct? (y/n/exit) +``` + +### WEBAPP, STORAGE and BROKER passwords + +You can use the option of random password generation when prompted for and `WIS2BOX_WEBAPP_PASSWORD`, `WIS2BOX_STORAGE_PASSWORD`, `WIS2BOX_BROKER_PASSWORD` and define your own. + +Don't worry about remembering these passwords, they will be stored in the `wis2box.env` file in your wis2box-1.0b8 directory. + +### Review `wis2box.env` + +Once the scripts is completed check the contents of the `wis2box.env` file in your current directory: + +```bash +cat ~/wis2box-1.0b8/wis2box.env +``` + +Or check the content of the file via WinSCP. + +!!! question + + What is the value of WISBOX_BASEMAP_URL in the wis2box.env file? + +??? success "Click to reveal answer" + + The default value for WIS2BOX_BASEMAP_URL is `https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png`. + + This URL refers to the OpenStreetMap tile server. If you want to use a different map provider, you can change this URL to point to a different tile server. + +!!! question + + What is the value of the WIS2BOX_STORAGE_DATA_RETENTION_DAYS environment variable in the wis2box.env file? + +??? success "Click to reveal answer" + + The default value for WIS2BOX_STORAGE_DATA_RETENTION_DAYS is 30 days. You can change this value to a different number of days if you wish. + + The wis2box-management container runs a cronjob on a daily basis to remove data older than the number of days defined by WIS2BOX_STORAGE_DATA_RETENTION_DAYS from the `wis2box-public` bucket and the API backend: + + ```{.copy} + 0 0 * * * su wis2box -c "wis2box data clean --days=$WIS2BOX_STORAGE_DATA_RETENTION_DAYS" + ``` + +!!! note + + The `wis2box.env` file contains environment variables defining the configuration of your wis2box. For more information consult the [wis2box-documentation](https://docs.wis2box.wis.wmo.int/en/latest/reference/configuration.html). + + Do not edit the `wis2box.env` file unless you are sure of the changes you are making. Incorrect changes can cause your wis2box to stop working. + + Do not share the contents of your `wis2box.env` file with anyone, as it contains sensitive information such as passwords. + + +## Start wis2box + +Ensure you are in the directory containing the wis2box software stack definition files: + +```{.copy} +cd ~/wis2box-1.0b8 +``` + +Start wis2box with the following command: + +```{.copy} +python3 wis2box-ctl.py start +``` + +Inspect the status with the following command: + +```{.copy} +python3 wis2box-ctl.py status +``` + +Repeat this command until all services are up and running. + +!!! note "wis2box and Docker" + wis2box runs as a set of Docker containers managed by docker-compose. + + The services are defined in the various `docker-compose*.yml` which can be found in the `~/wis2box-1.0b8/` directory. + + The Python script `wis2box-ctl.py` is used to run the underlying Docker Compose commands that control the wis2box services. + + You don't need to know the details of the Docker containers to run the wis2box software stack, but you can inspect the `docker-compose*.yml` and files to see how the services are defined. If you are interested in learning more about Docker, you can find more information in the [Docker documentation](https://docs.docker.com/). + +To login to the wis2box-management container, use the following command: + +```{.copy} +python3 wis2box-ctl.py login +``` + +Inside the wis2box-management container you can run various commands to manage your wis2box, such as: + +- `wis2box auth add-token --path processes/wis2box` : to create an authorization token for the `processes/wis2box` endpoint +- `wis2box data clean --days=` : to clean up data older than a certain number of days from the `wis2box-public` bucket + +To exit the container and go back to the host machine, use the following command: + +```{.copy} +exit +``` + +Run the following command to see the docker containers running on your host machine: + +```{.copy} +docker ps +``` + +You should see the following containers running: + +- wis2box-management +- wis2box-api +- wis2box-minio +- wis2box-webapp +- wis2box-auth +- wis2box-ui +- wis2downloader +- elasticsearch +- elasticsearch-exporter +- nginx +- mosquitto +- prometheus +- grafana +- loki + +These containers are part of the wis2box software stack and provide the various services required to run the wis2box. + +Run the following command to see the docker volumes running on your host machine: + +```{.copy} +docker volume ls +``` + +You should see the following volumes: + +- wis2box_project_auth-data +- wis2box_project_es-data +- wis2box_project_htpasswd +- wis2box_project_minio-data +- wis2box_project_prometheus-data +- wis2box_project_loki-data + +As well as some anonymous volumes used by the various containers. + +The volumes starting with `wis2box_project_` are used to store persistent data for the various services in the wis2box software stack. + +## wis2box API + +The wis2box contains an API (Application Programming Interface) that provide data access and processes for interactive visualization, data transformation and publication. + +Open a new tab and navigate to the page `http:///oapi`. + +wis2box-api.png + +This is the landing page of the wis2box API (running via the **wis2box-api** container). + +!!! question + + What collections are currently available? + +??? success "Click to reveal answer" + + To view collections currently available through the API, click `View the collections in this service`: + + wis2box-api-collections.png + + The following collections are currently available: + + - Stations + - Data notifications + - Discovery metadata + + +!!! question + + How many data notifications have been published? + +??? success "Click to reveal answer" + + Click on "Data notifications", then click on `Browse through the items of "Data Notifications"`. + + You will note that the page says "No items" as no Data notifications have been published yet. + +## wis2box webapp + +Open a web browser and visit the page `http:///wis2box-webapp`. + +You will see a pop-up asking for your username and password. Use the default username `wis2box-user` and the `WIS2BOX_WEBAPP_PASSWORD` defined in the `wis2box.env` file and click "Sign in": + +!!! note + + Check you wis2box.env for the value of your WIS2BOX_WEBAPP_PASSWORD. You can use the following command to check the value of this environment variable: + + ```{.copy} + cat ~/wis2box-1.0b8/wis2box.env | grep WIS2BOX_WEBAPP_PASSWORD + ``` + +Once logged in, you move your mouse to the menu on the left to see the options available in the wis2box web application: + +wis2box-webapp-menu.png + +This is the wis2box web application to enable you to interact with your wis2box: + +- create and manage datasets +- update/review your station metadata +- ingest ASCII and CSV data +- monitor notifications published on your wis2box-broker + +We will use this web application in a later session. + +## wis2box-broker + +Open the MQTT Explorer on your computer and prepare a new connection to connect to your broker (running via the **wis2box-broker** container). + +Click `+` to add a new connection: + +mqtt-explorer-new-connection.png + +You can click on the 'ADVANCED' button and verify you have subscriptions to the the following topics: + +- `#` +- `$SYS/#` + +mqtt-explorer-topics.png + +!!! note + + The `#` topic is a wildcard subscription that will subscribe to all topics published on the broker. + + The messages published under the `$SYS` topic are system messages published by the mosquitto service itself. + +Use the following connection details, making sure to replace the value of `` with your hostname and `` with the value from your `wis2box.env` file: + +- **Protocol: mqtt://** +- **Host: ``** +- **Port: 1883** +- **Username: wis2box** +- **Password: ``** + +!!! note + + You can check your wis2box.env for the value of your WIS2BOX_BROKER_PASSWORD. You can use the following command to check the value of this environment variable: + + ```{.copy} + cat ~/wis2box-1.0b8/wis2box.env | grep WIS2BOX_BROKER_PASSWORD + ``` + + Note that this your **internal** broker password, the Global Broker will use different (read-only) credentials to subscribe to your broker. Never share this password with anyone. + +Make sure to click "SAVE" to store your connection details. + +Then click "CONNECT" to connect to your **wis2box-broker**. + +mqtt-explorer-wis2box-broker.png + +Once you are connected, verify that your the internal mosquitto statistics being published by your broker under the `$SYS` topic: + +mqtt-explorer-sys-topic.png + +Keep the MQTT Explorer open, as we will use it to monitor the messages published on the broker. + +## MinIO UI + +Open a web browser and visit the page `http://:9001`: + +minio-ui.png + +This is the MinIO UI (running via the **wis2box-storage** container). + +The username and password are defined in the `wis2box.env` file in your wis2box data directory by the environment variables `WIS2BOX_STORAGE_USERNAME` and `WIS2BOX_STORAGE_PASSWORD`. The default username is `wis2box`. + +!!! note + + You can check your wis2box.env for the value of your WIS2BOX_STORAGE_PASSWORD. You can use the following command to check the value of this environment variable: + + ```{.copy} + cat ~/wis2box-1.0b8/wis2box.env | grep WIS2BOX_STORAGE_PASSWORD + ``` + + Note that these are the read-write credentials for your MinIO instance. Never share these credentials with anyone. The Global Services can only download data from your MinIO instance using the web-proxy on the wis2box-public bucket. + +Try to login to your MinIO UI. You will see that there 3 buckets already defined: + +- `wis2box-incoming`: used to receive incoming data +- `wis2box-public`: used to store data that is made available in the WIS2 notifications, the content of this bucket is proxied as `/data` on your `WIS2BOX_URL` via the nginx container +- `wis2box-archive`: used to archive data from `wis2box-incoming` on a daily basis + +minio-ui-buckets.png + +!!! note + + The **wis2box-storage** container (provided by MinIO) will send a notification on the **wis2box-broker** when data is received. The **wis2box-management** container is subscribed to all messages on `wis2box/#` and will receive these notifications, triggering the data plugins defined in the datasets. + +### Using the MinIO UI to test data ingestion + +First click on "browse" for the `wis2box-incoming` bucket, then click on "Create new path": + +minio-ui-create-new-path.png + +And enter the path `/testing/upload/`: + +minio-ui-create-path.png + +Right-click on this link and select 'save link as' : [randomfile.txt](/sample-data/randomfile.txt), to download the file to your local machine. + +Then click on "Upload" and select the randomfile.txt you downloaded from your local machine and upload it to the `wis2box-incoming` bucket + +You should now see the file in the `wis2box-incoming` bucket: + +minio-ui-upload-file.png + +!!! question + + Did you see a message published on your MQTT broker in MQTT Explorer? + +??? success "Click to reveal answer" + + If you are connected to your wis2box-broker, you should note that a message has been published on the `wis2box/storage` topic: + + mqtt-explorer-wis2box-storage.png + + This is how MinIO informs the wis2box-management service that there is new data to process. + + Note: this is **not** a WIS2 notification, but an internal message between components within the wis2box software stack. + +The wis2box is not yet configured to process any files you upload, so the file will not be processed and no WIS2 notification will be published. + +We will check the error message produced by the wis2box-management container in the Grafana dashboard in the next step. + +## Grafana UI + +Open a web browser and visit the page `http://:3000`: + +wis2box-grafana-ui.png + +This is the Grafana UI, where you can view the wis2box workflow monitoring dashboard. You can also access the logs of the various containers in the wis2box software stack via the 'Explore' option in the menu. + +!!! question + + Can you find the error message produced by the wis2box-management container when you uploaded the file to the `wis2box-incoming` bucket? + +??? success "Click to reveal answer" + + You will should see an error message indicating that the wis2box could not process the file you uploaded. + + This is expected as we have not yet configured any datasets in the wis2box, you will learn how to do this in the next practical session. + +## Conclusion + +!!! success "Congratulations!" + In this practical session, you learned how to: + + - run the `wis2box-create-config.py` script to create the initial configuration + - start wis2box and check the status of its components + - connect to the MQTT broker on your student-VM using MQTT Explorer + - access the wis2box-UI, wis2box-webapp and wis2box-API in a browser + - upload a file to the "wis2box-incoming"-bucket in the MinIO UI + - view the Grafana dashboard to monitor the wis2box workflow diff --git a/documentation/docs/practical-sessions/initializing-wis2box.md b/documentation/docs/practical-sessions/initializing-wis2box.md deleted file mode 100644 index 4506e4c5..00000000 --- a/documentation/docs/practical-sessions/initializing-wis2box.md +++ /dev/null @@ -1,373 +0,0 @@ ---- -title: Initializing wis2box ---- - -# Initializing wis2box - -!!! abstract "Learning outcomes" - - By the end of this practical session, you will be able to: - - - run the `wis2box-create-config.py` script to create the initial configuration - - start wis2box and check the status of its components - - access the **wis2box-webapp**, API, MinIO UI and Grafana dashboard in a browser - - connect to the local **wis2box-broker** using MQTT Explorer - -!!! note - - The current training materials are using wis2box-1.0b5. - - See [accessing-your-student-vm](accessing-your-student-vm.md) for instructions on how to download and install the wis2box software stack if you are running this training outside of a local training session. - -## Preparation - -Login to your designated VM with your username and password and ensure you are in the `wis2box-1.0b5` directory: - -```bash -cd ~/wis2box-1.0b5 -``` - -## `wis2box-create-config.py` - -The `wis2box-create-config.py` script can be used to create the initial configuration of your wis2box. - -It will ask you a set of question to help setup your configuration. - -You will be able to review and update the configuration files after the script has completed. - -Run the script as follows: - -```bash -python3 wis2box-create-config.py -``` - -### wis2box-data directory - -The script will ask you to enter the directory where your configuration and data will be stored. - -We recommend you use the directory `wis2box-data` in your home directory to store your configuration and data. -Note that you need to define the full path to this directory. - -For example if your username is `mlimper`, the full path to the directory is `/home/mlimper/wis2box-data`: - -```{.copy} -mlimper@student-vm-mlimper:~/wis2box-1.0b5$ python3 wis2box-create-config.py -Please enter the directory on the host where wis2box-configuration-files are to be stored: -/home/mlimper/wis2box-data -Configuration-files will be stored in the following directory: - /home/mlimper/wis2box-data -Is this correct? (y/n/exit) -y -The directory /home/mlimper/wis2box-data has been created. -``` - -### wis2box URL - -Next, you will be asked to enter the URL for your wis2box. This is the URL that will be used to access the wis2box web application, API and UI. - -Please use `http://` as the URL. Remember that your hostname is defined by your `username.wis2.training` - -```{.copy} -Please enter the URL of the wis2box: - For local testing the URL is http://localhost - To enable remote access, the URL should point to the public IP address or domain name of the server hosting the wis2box. -http://mlimper.wis2.training -The URL of the wis2box will be set to: - http://mlimper.wis2.training -Is this correct? (y/n/exit) -``` - -### STORAGE and BROKER passwords - -You can use the option of random password generation when prompted for `WIS2BOX_STORAGE_PASSWORD` and `WIS2BOX_BROKER_PASSWORD` or define your own. - -Don't worry about remembering these passwords, they will be stored in the `wis2box.env` file in your wis2box-1.0b5 directory. - -### Country code and centre-id - -Next you will be asked for the 3-letter ISO code for your country and centre-id for your wis2box. The centre-id can be a string of your choosing for the purpose of this training. - -```{.copy} -Please enter your 3-letter ISO country code: -nld -Please enter the centre-id for your wis2box: -maaike_test -The country-code will be set to: - nld -The centre-id will be set to: - maaike_test -Is this correct? (y/n/exit) -``` - -### Discovery metadata - -Next you will answer a set of question to generate discovery metadata templates for your wis2box. The answers do not need to be correct for the purpose of this training. - -```{.copy} -******************************************************************************** -Creating initial configuration for surface and upper-air data. -******************************************************************************** -Please enter the email address of the wis2box administrator: -mlimper@wmo.int -The email address of the wis2box administrator will be set to: - mlimper@wmo.int -Is this correct? (y/n/exit) -n -Please enter the email address of the wis2box administrator: -me@gmail.com -The email address of the wis2box administrator will be set to: - me@gmail.com -Is this correct? (y/n/exit) -y -Please enter the name of your organization: -Maaike-TEST -Your organization name will be set to: - Maaike-TEST -Is this correct? (y/n/exit) -y -Getting bounding box for "nld". -bounding box: -68.6255319,11.825,7.2274985,53.744395. -Do you want to use this bounding box? (y/n/exit) -y -Created new metadata file: /home/mlimper/wis2box-data/metadata/discovery/metadata-synop.yml -Created new metadata file: /home/mlimper/wis2box-data/metadata/discovery/metadata-temp.yml -``` - -We will review the discovery metadata templates in a later session. - -### review configuration - -Once the scripts is completed check the contents of the `wis2box.env` file in your current directory: - -```bash -cat ~/wis2box-1.0b5/wis2box.env -``` - -Or check the content of the file via WinSCP. - -!!! question - - What is the value of the WIS2BOX_STORAGE_DATA_RETENTION_DAYS environment variable in the wis2box.env file? - -??? success "Click to reveal answer" - - The default value for WIS2BOX_STORAGE_DATA_RETENTION_DAYS is 30 days. You can change this value to a different number of days if you wish. - - The wis2box-management container runs a cronjob on a daily basis to remove data older than the number of days defined by WIS2BOX_STORAGE_DATA_RETENTION_DAYS from the `wis2box-public` bucket and the API backend: - - ```{.copy} - 0 0 * * * su wis2box -c "wis2box data clean --days=$WIS2BOX_STORAGE_DATA_RETENTION_DAYS" - ``` - -!!! note - - The `wis2box.env` file contains environment variables defining the configuration of your wis2box. For more information consult the [wis2box-documentation](https://docs.wis2box.wis.wmo.int/en/latest/reference/configuration.html) - -Next, check the contents of the `data-mappings.yml` file in your wis2box data directory: - -```bash -cat ~/wis2box-data/data-mappings.yml -``` - -Or check the content of the data-mappings.yml via WinSCP by browsing to the new directory 'wis2box-data' (click refresh if you don't see it yet) - -!!! question - - How many different keys are defined for `data` in the `data-mappings.yml` file? - -??? success "Click to reveal answer" - - There are 2 different 'data'-keys defined in the data-mappings.yml file, one for surface-based observations and one for upper-air observations: - - - nld.maaike_test.data.core.weather.surface-based-observations.synop - - nld.maaike_test.data.core.weather.upper-air-observations.temp - - The country-code and centre-id will be different from the example above, they will be set to the values you entered during the `wis2box-create-config.py` script. - - You can also note that different 'plugins' are defined for the different data types. The use of these plugins in the wis2box data pipeline architecture will be discussed in a later session. - - -!!! note - - The `data-mappings.yml` file defines the plugins used to transform your data. For more information see [data pipeline plugins in the wis2box-documentation](https://docs.wis2box.wis.wmo.int/en/latest/reference/running/data-pipeline-plugins.html) - - -## wis2box start and status - -Ensure you are in the directory containing the wis2box software stack: - -```{.copy} -cd ~/wis2box-1.0b5 -``` - -Start wis2box with the following command: - -```{.copy} -python3 wis2box-ctl.py start -``` - -Inspect the status with the following command: - -```{.copy} -python3 wis2box-ctl.py status -``` - -Repeat this command until all services are up and running. - - -!!! note "wis2box and Docker" - wis2box runs as a set of Docker containers managed by docker-compose. - - The services are defined in the various `docker-compose*.yml` which can be found in the `~/wis2box-1.0b5/` directory. - - The Python script `wis2box-ctl.py` is used to run the underlying Docker Compose commands that control the wis2box services. - -## wis2box API - -Open a new tab and navigate to the page `http:///oapi`. - -wis2box-api.png - -This is the wis2box API (running via the **wis2box-api** container). - -!!! question - - What collections are currently available? - -??? success "Click to reveal answer" - - To view collections currently available through the API, click `View the collections in this service`: - - wis2box-api-collections.png - - The following collections are currently available: - - - Discovery metadata - - Station metadata - - Data notifications - -!!! question - - How many data notifications have been published? - -??? success "Click to reveal answer" - - Click on "Data notifications", then click on `Browse through the items of "Data Notifications"`. - - You will note that the page says "No items" as no Data notifications have been published yet. - -## wis2box webapp - -Open a web browser and visit the page `http:///wis2box-webapp`: - -wis2box-webapp.png - -This is the (new) wis2box web application to enable you to interact with your wis2box: - -- ingest ASCII and CSV data -- update/review your station metadata -- monitor notifications published on your wis2box-broker - -We will use this web application in a later session. - -## wis2box UI - -Open a web browser and visit the page `http://`: - -wis2box-ui.png - -The wis2box UI will display your configured datasets. The UI is currently empty, as datasets have not yet been configured. - -## wis2box-broker - -Open the MQTT Explorer on your computer and prepare a new connection to connect to your broker (running via the **wis2box-broker** container). - -Click `+` to add a new connection: - -mqtt-explorer-new-connection.png - -Click on the 'ADVANCED' button make sure you have subscriptions to the the following topics: - -- `$SYS/#` -- `origin/#` -- `wis2box/#` - -mqtt-explorer-topics.png - -!!! note - - The messages published under the `$SYS` topic are system messages published by the mosquitto service itself. - - The messages published under topics starting with `origin/a/wis2/#` are the WIS2 data notifications published by the **wis2box-broker**. - - The messages published under topics starting with `wis2box` are internal messages between the various components of the wis2box software stack. - -Use the following connection details, making sure to replace the value of `` with your hostname and `` with the value from your `wis2box.env` file: - -- **Protocol: mqtt://** -- **Host: ``** -- **Port: 1883** -- **Username: wis2box** -- **Password: ``** - -!!! note - - Check you wis2box.env for the value of your WIS2BOX_BROKER_PASSWORD. - -Make sure to click "SAVE" to store your connection details. - -Then click "CONNECT" to connect to your **wis2box-broker**. - -mqtt-explorer-wis2box-broker.png - -Once you are connected, you should see statistics being published by your broker on the `$SYS/#`. - -Later during the training you will use the MQTT connection you saved to view notifications published by your wis2box-broker. - -## MinIO UI - -Open a web browser and visit the page `http://:9001`: - -minio-ui.png - -This is the MinIO UI (running via the **wis2box-storage** container). - -The username and password are defined in the `wis2box.env` file in your wis2box data directory by the environment variables `WIS2BOX_STORAGE_USERNAME` and `WIS2BOX_STORAGE_PASSWORD`. - -Use the command below to check the values of these environment variables from the command line in your SSH session: - -```{.copy} -cat ~/wis2box-1.0b5/wis2box.env -``` - -Or check the content of the file via WinSCP. - -Try to login to your MinIO UI. You will see that there 3 buckets already defined: - -- `wis2box-incoming`: used to receive incoming data -- `wis2box-public`: used to store data that is made available in the WIS2 notifications, the content of this bucket is proxied as `/data` on your `WIS2BOX_URL` via the nginx container -- `wis2box-archive`: used to archive data from `wis2box-incoming` on a daily basis - -minio-ui-buckets.png - -!!! note - - The **wis2box-storage** container will send a notification on the **wis2box-broker** when data is received. The **wis2box-management** container is subscribed to all messages on `wis2box/#` and will receive these notifications, triggering the data pipelines defined in your `data-mappings.yml`. - -## Grafana UI - -Open a web browser and visit the page `http://:3000`: - -wis2box-grafana-ui.png - -This is the Grafana UI, where you can view the wis2box workflow monitoring dashboard. You can also access the logs of the various containers in the wis2box software stack via the 'Explore' option in the menu. - -## Conclusion - -!!! success "Congratulations!" - In this practical session, you learned how to: - - - run the `wis2box-create-config.py` script to create the initial configuration - - start wis2box and check the status of its components - - access the **wis2box-webapp**, API, MinIO UI and Grafana dashboard in a browser - - connect to the **wis2box-broker** using MQTT Explorer diff --git a/documentation/docs/practical-sessions/monitoring-wis2-notifications.md b/documentation/docs/practical-sessions/monitoring-wis2-notifications.md new file mode 100644 index 00000000..91965607 --- /dev/null +++ b/documentation/docs/practical-sessions/monitoring-wis2-notifications.md @@ -0,0 +1,186 @@ +--- +title: Monitoring WIS2 Notifications +--- + +# Monitoring WIS2 Notifications + +!!! abstract "Learning outcomes" + + By the end of this practical session, you will be able to: + + - trigger the wis2box workflow by uploading data in MinIO using the `wis2box data ingest` command + - view warnings and errors displayed in the Grafana dashboard + - check the content of the data being published + +## Introduction + +The **Grafana dashboard** uses data from Prometheus and Loki to display the status of your wis2box. Prometheus store time-series data from the metrics collected, while Loki store the logs from the containers running on your wis2box instance. This data allows you to check how much data is received on MinIO and how many WIS2 notifications are published, and if there are any errors detected in the logs. + +To see the content of the WIS2 notifications that are being published on different topics of your wis2box you can use the 'Monitor' tab in the **wis2box-webapp**. + +## Preparation + +This section will use the "surface-based-observations/synop" dataset previously created in the [Configuring datasets in wis2box](/practical-sessions/configuring-wis2box-datasets) practical session. + +Login to your student VM using your SSH client (PuTTY or other). + +Make sure wis2box is up and running: + +```bash +cd ~/wis2box-1.0b8/ +python3 wis2box-ctl.py start +python3 wis2box-ctl.py status +``` + +Make sure your have MQTT Explorer running and connected to your instance using the public credentials `everyone/everyone` with a subscription to the topic `origin/a/wis2/#`. + +Make sure you have access to the MinIO web interface by going to `http://:9000` and you are logged (using `WIS2BOX_STORAGE_USERNAME` and `WIS2BOX_STORAGE_PASSWORD` from your `wis2box.env` file). + +Make sure you have a web browser open with the Grafana dashboard for your instance by going to `http://:3000`. + +## Ingesting some data + +Please execute the following commands from your SSH-client session: + +Copy the sample data file `aws-example.csv` to the the directory you defined as the `WI2BOX_HOST_DATADIR` in your `wis2box.env` file. + +```bash +cp ~/exercise-materials/monitoring-exercises/aws-example.csv ~/wis2box-data/ +``` + +Make sure you are in the `wis2box-1.0b8` directory and login to the **wis2box-management** container: + +```bash +cd ~/wis2box-1.0b8 +python3 wis2box-ctl.py login +``` + +Verify the sample data is available in the directory `/data/wis2box/` within the **wis2box-management** container: + +```bash +ls -lh /data/wis2box/aws-example.csv +``` + +!!! note + The `WIS2BOX_HOST_DATADIR` is mounted as `/data/wis2box/` inside the wis2box-management container by the `docker-compose.yml` file included in the `wis2box-1.0b8` directory. + + This allows you to share data between the host and the container. + +!!! question "Exercise 1: ingesting data using `wis2box data ingest`" + + Execute the following command to ingest the sample data file `aws-example.csv` to your wis2box-instance: + + ```bash + wis2box data ingest -p /data/wis2box/aws-example.csv --metadata-id urn:wmo:md:not-my-centre:core.surface-based-observations.synop + ``` + + Was the data successfully ingested? If not, what was the error message and how can you fix it? + +??? success "Click to reveal answer" + + You will see the following output: + + ```bash + Error: metadata_id=urn:wmo:md:not-my-centre:core.surface-based-observations.synop not found in data mappings + ``` + + The error message indicates that the metadata identifier you provided does not match any of the datasets you have configured in your wis2box-instance. + + Provide the correct metadata-id that matches the dataset you created in the previous practical session and repeat the data ingest command until you should see the following output: + + ```bash + Processing /data/wis2box/aws-example.csv + Done + ``` + +Go to the MinIO console in your browser and check if the file `aws-example.csv` was uploaded to the `wis2box-incoming` bucket. You should see there is a new directory with the name of the dataset you provided in the `--metadata-id` option: + +minio-wis2box-incoming-dataset-folder + +!!! note + The `wis2box data ingest` command uploaded the file to the `wis2box-incoming` bucket in MinIO in a directory named after the metadata identifier you provided. + +Go to the Grafana dashboard in your browser and check the status of the data ingest. + +!!! question "Exercise 2: check the status of the data ingest" + + Go to the Grafana dashboard in your browser and check the status of the data ingest. + + Was the data successfully ingested? + +??? success "Click to reveal answer" + The panel at the bottom of the Grafana home dashboard reports the following warnings: + + `WARNING - input=aws-example.csv warning=Station 0-20000-0-60355 not in station list; skipping` + `WARNING - input=aws-example.csv warning=Station 0-20000-0-60360 not in station list; skipping` + + This warning indicates that the stations are not defined in the station list of your wis2box. No WIS2 notifications will be published for this station until you add it to the station list and associate it with the topic for your dataset. + +!!! question "Exercise 3: add the test stations and repeat the data ingest" + + Add the stations to your wis2box using the station editor in **wis2box-webapp**, and associate the stations with the topic for your dataset. + + Now re-upload the sample data file `aws-example.csv` to the same path in MinIO you used in the previous exercise. + + Check the Grafana dashboard, are there any new errors or warnings ? How can you see that the test data was successfully ingested and published? + +??? success "Click to reveal answer" + + You can check the charts on the Grafana home dashboard to see if the test data was successfully ingested and published. + + If successful, you should see the following: + + grafana_success + +!!! question "Exercise 4: check the MQTT broker for WIS2 notifications" + + Go to the MQTT Explorer and check if you can see the WIS2 Notification Message for the data you just ingested. + + How many WIS2 data notifications were published by your wis2box? + + How do you access the content of the data being published? + +??? success "Click to reveal answer" + + You should see 6 WIS2 data notifications published by your wis2box. + + To access the content of the data being published, you can expand the topic structure to see the different levels of the message until you reach the last level and review message content of one of the messages. + + The message content has a "links" section with a "rel" key of "canonical" and a "href" key with the URL to download the data. The URL will be in the format `http:///data/...`. + + Note that the data-format is BUFR and you will need a BUFR parser to view the content of the data. The BUFR format is a binary format used by meteorological services to exchange data. The data-plugins inside wis2box transformed the data from CSV to BUFR before publishing it. + +## Viewing the data content you have published + +You can use the **wis2box-webapp** to view the content of the WIS2 data notifications that have been published by your wis2box. + +Open the **wis2box-webapp** in your browser by navigating to `http:///wis2box-webapp` and select the **Monitoring** tab: + +wis2box-webapp-monitor + +In the monitoring-tab select your dataset-id and click "UPDATE" + +??? question "Exercise 5: view the WIS2 notifications in the wis2box-webapp" + + How many WIS2 data notifications were published by your wis2box? + + What is the air-temperature reported in the last notification at the station with the WIGOS-identifier=0-20000-0-60355? + +??? success "Click to reveal answer" + + If you have successfully ingested the test data, you should see 6 WIS2 data notifications published by your wis2box. + + To see the air-temperature measured for the station with WIGOS-identifier=0-20000-0-60355, click on the "INSPECT"-button next to the file for that station to open a pop-up window displaying the parsed content of the data file. The air-temperature measured at this station was 25.0 degrees Celsius. + +!!! Note + The wis2box-api container includes tools to parse BUFR files and display the content in a human-readable format. This is a not a core requirements for the WIS2.0 implementation, but was included in the wis2box to aid data publishers in checking the content of the data they are publishing. + +## Conclusion + +!!! success "Congratulations!" + In this practical session, you learned how to: + + - trigger the wis2box workflow by uploading data in MinIO using the `wis2box data ingest` command + - view the WIS2 notifications published by your wis2box in the Grafana dashboard and MQTT Explorer + - check the content of the data being published using the **wis2box-webapp** + diff --git a/documentation/docs/practical-sessions/querying-data-using-the-wis2box-api.md b/documentation/docs/practical-sessions/querying-data-using-the-wis2box-api.en.md similarity index 98% rename from documentation/docs/practical-sessions/querying-data-using-the-wis2box-api.md rename to documentation/docs/practical-sessions/querying-data-using-the-wis2box-api.en.md index 802af6d6..48ea7761 100644 --- a/documentation/docs/practical-sessions/querying-data-using-the-wis2box-api.md +++ b/documentation/docs/practical-sessions/querying-data-using-the-wis2box-api.en.md @@ -77,7 +77,7 @@ Click on the 'Queryables' link. ??? success "Click to reveal answer" The `wigos_station_identifer` is the correct queryable. -Navigate to the previous page (i.e. `http:///oapi/collections/urn:x-wmo:md:mwi:mwi_met_centre:surface-weather-observations`) +Navigate to the previous page (i.e. `http:///oapi/collections/urn:wmo:md:mwi:mwi_met_centre:surface-weather-observations`) Click on the 'Browse' link. diff --git a/documentation/docs/sample-data/A_SMRO01YRBK171200_C_EDZW_20240717120502.txt b/documentation/docs/sample-data/A_SMRO01YRBK171200_C_EDZW_20240717120502.txt new file mode 100644 index 00000000..f70a4fb1 --- /dev/null +++ b/documentation/docs/sample-data/A_SMRO01YRBK171200_C_EDZW_20240717120502.txt @@ -0,0 +1,15 @@ +SMRO01 YRBK 171200 + +AAXX 17121 + +15015 01597 71702 10057 20036 39390 42628 50004 60021 78082 87300 333 + +4/000 + +55304 0//// 20643 3//// 69977 91003 91108= + +15020 02597 61303 10104 20040 39783 49976 58007 60001 83570 333 4/000 + +55308 + +0//// 21085 3//// 60007 91005 91106= \ No newline at end of file diff --git a/documentation/docs/sample-data/randomfile.txt b/documentation/docs/sample-data/randomfile.txt new file mode 100644 index 00000000..e514e0ac --- /dev/null +++ b/documentation/docs/sample-data/randomfile.txt @@ -0,0 +1 @@ +this is just a random file \ No newline at end of file diff --git a/documentation/mkdocs.yml b/documentation/mkdocs.yml index 3f798a78..36dd698f 100644 --- a/documentation/mkdocs.yml +++ b/documentation/mkdocs.yml @@ -15,18 +15,21 @@ nav: - Connecting to WIS2 over MQTT: practical-sessions/connecting-to-wis2-over-mqtt.md - Accessing your student VM: practical-sessions/accessing-your-student-vm.md - Initializing wis2box: practical-sessions/initializing-wis2box.md - - Configuring WIS2 discovery metadata: practical-sessions/configuring-wis2-discovery-metadata.md + - Configuring datasets in wis2box: practical-sessions/configuring-wis2box-datasets.md - Configuring station metadata: practical-sessions/configuring-station-metadata.md - - Converting SYNOP data to BUFR: practical-sessions/converting-synop-data-to-bufr.md + - Monitoring WIS2 notifications: practical-sessions/monitoring-wis2-notifications.md + - Converting SYNOP data to BUFR from the command line: practical-sessions/converting-synop-data-to-bufr.md + - Converting SYNOP data to BUFR using the wis2box-webapp: practical-sessions/converting-synop-data-to-bufr-form.md - Converting CSV data to BUFR: practical-sessions/converting-csv-data-to-bufr.md - - BUFR command line tools: practical-sessions/bufr-command-line-tools.md - Automating data ingestion: practical-sessions/automating-data-ingestion.md - - Downloading and finding data from WIS2: practical-sessions/downloading-and-finding-data-from-wis2.md - - Querying data using the wis2box API: practical-sessions/querying-data-using-the-wis2box-api.md + - Adding GTS-headers to WIS2 notifications: practical-sessions/adding-gts-headers-to-wis2-notifications.md + - Setting up a recommended dataset with access control: practical-sessions/datasets-with-access-control.md + - Downloading data from WIS2: practical-sessions/downloading-data-from-wis2.md + #- Querying data using the wis2box API: practical-sessions/querying-data-using-the-wis2box-api.md + #- Working with BUFR data: practical-sessions/bufr-command-line-tools.md - Cheatsheets: - Linux: cheatsheets/linux.md - Docker: cheatsheets/docker.md - - YAML: cheatsheets/yaml.md - WIS2 in a box: cheatsheets/wis2box.md strict: true @@ -46,6 +49,22 @@ theme: plugins: - search - table-reader + - i18n: + docs_structure: suffix + languages: + - locale: en + default: true + name: English + build: true + - locale: es + name: Español + build: true + - locale: fr + name: Français + build: true + - locale: ru + name: Русский + build: true - print-site markdown_extensions: diff --git a/documentation/requirements.txt b/documentation/requirements.txt index 73487efa..36f448ca 100644 --- a/documentation/requirements.txt +++ b/documentation/requirements.txt @@ -1,4 +1,5 @@ mkdocs mkdocs-material mkdocs-print-site-plugin -mkdocs-table-reader-plugin \ No newline at end of file +mkdocs-table-reader-plugin +mkdocs-static-i18n \ No newline at end of file diff --git a/environment/setup_student_vm.sh b/environment/setup_student_vm.sh index 3992978e..96853f62 100644 --- a/environment/setup_student_vm.sh +++ b/environment/setup_student_vm.sh @@ -23,37 +23,24 @@ read -p "Continue? (Y/N): " confirm && [[ $confirm == [yY] || $confirm == [yY][e # execute commands over ssh # create the user, if not already existing -ssh wmo_admin@`echo $HOST_IP` "sudo useradd "$USERNAME" -m -G docker --shell /bin/bash" +ssh -o StrictHostKeyChecking=no wmo_admin@`echo $HOST_IP` "sudo useradd "$USERNAME" -m -G docker --shell /bin/bash" # set the initial password to "wis2training" -ssh wmo_admin@`echo $HOST_IP` "echo "$USERNAME":wis2training | sudo chpasswd" +ssh -o StrictHostKeyChecking=no wmo_admin@`echo $HOST_IP` "echo "$USERNAME":wis2training | sudo chpasswd" # rename the hostname to student-vm- -ssh wmo_admin@`echo $HOST_IP` "sudo hostnamectl set-hostname student-vm-`echo $USERNAME`" +ssh -o StrictHostKeyChecking=no wmo_admin@`echo $HOST_IP` "sudo hostnamectl set-hostname student-vm-`echo $USERNAME`" -# copy the latest wis2box-setup-1.0b5.zip to the student-vm -ssh wmo_admin@`echo $HOST_IP` "wget https://github.com/wmo-im/wis2box/releases/download/1.0b5/wis2box-setup-1.0b5.zip -O /tmp/wis2box-setup-1.0b5.zip" -# unzip the wis2box-setup-1.0b5.zip -ssh wmo_admin@`echo $HOST_IP` "sudo unzip -o /tmp/wis2box-setup-1.0b5.zip -d /home/`echo $USERNAME`/" -# remove the wis2box-setup-1.0b5.zip -ssh wmo_admin@`echo $HOST_IP` "rm -rf /tmp/wis2box-setup-1.0b5.zip" +# copy the latest wis2box-setup-1.0b8.zip to the student-vm +ssh -o StrictHostKeyChecking=no wmo_admin@`echo $HOST_IP` "wget https://github.com/wmo-im/wis2box/releases/download/1.0b8/wis2box-setup-1.0b8.zip -O /tmp/wis2box-setup-1.0b8.zip" +# unzip the wis2box-setup-1.0b8.zip +ssh -o StrictHostKeyChecking=no wmo_admin@`echo $HOST_IP` "sudo unzip -o /tmp/wis2box-setup-1.0b8.zip -d /home/`echo $USERNAME`/" +# remove the wis2box-setup-1.0b8.zip +ssh -o StrictHostKeyChecking=no wmo_admin@`echo $HOST_IP` "rm -rf /tmp/wis2box-setup-1.0b8.zip" # copy the latest exercise materials to the student-vm -ssh wmo_admin@`echo $HOST_IP` "wget https://training.wis2box.wis.wmo.int/exercise-materials.zip -O /tmp/exercise-materials.zip" +ssh -o StrictHostKeyChecking=no wmo_admin@`echo $HOST_IP` "wget https://training.wis2box.wis2dev.io/exercise-materials.zip -O /tmp/exercise-materials.zip" # unzip the exercise-materials.zip -ssh wmo_admin@`echo $HOST_IP` "sudo unzip -o /tmp/exercise-materials.zip -d /home/`echo $USERNAME`/" +ssh -o StrictHostKeyChecking=no wmo_admin@`echo $HOST_IP` "sudo unzip -o /tmp/exercise-materials.zip -d /home/`echo $USERNAME`/" # remove the exercise-materials.zip -ssh wmo_admin@`echo $HOST_IP` "rm -rf /tmp/exercise-materials.zip" - -# create ftp.env -ssh wmo_admin@`echo $HOST_IP` "echo MYHOSTNAME=`echo $USERNAME`.wis2.training > /tmp/ftp.env" -ssh wmo_admin@`echo $HOST_IP` "echo FTP_USER=wis2box >> /tmp/ftp.env" -ssh wmo_admin@`echo $HOST_IP` "echo FTP_PASS=wis2box >> /tmp/ftp.env" -ssh wmo_admin@`echo $HOST_IP` "echo FTP_HOST=`echo $USERNAME`.wis2.training >> /tmp/ftp.env" -ssh wmo_admin@`echo $HOST_IP` "echo "WIS2BOX_STORAGE_ENDPOINT=http://`echo $USERNAME`.wis2.training:9000" >> /tmp/ftp.env" -ssh wmo_admin@`echo $HOST_IP` "echo WIS2BOX_STORAGE_USERNAME=minio >> /tmp/ftp.env" -ssh wmo_admin@`echo $HOST_IP` "echo WIS2BOX_STORAGE_PASSWORD=minio123 >> /tmp/ftp.env" -ssh wmo_admin@`echo $HOST_IP` "echo LOGGING_LEVEL=WARNING >> /tmp/ftp.env" -ssh wmo_admin@`echo $HOST_IP` "sudo cp /tmp/ftp.env /home/`echo $USERNAME`/wis2box-1.0b5/" -# remove /tmp/ftp.env -ssh wmo_admin@`echo $HOST_IP` "rm -rf /tmp/ftp.env" - -ssh wmo_admin@`echo $HOST_IP` "sudo chown -R `echo $USERNAME`:`echo $USERNAME` /home/`echo $USERNAME`" \ No newline at end of file +ssh -o StrictHostKeyChecking=no wmo_admin@`echo $HOST_IP` "rm -rf /tmp/exercise-materials.zip" + +ssh -o StrictHostKeyChecking=no wmo_admin@`echo $HOST_IP` "sudo chown -R `echo $USERNAME`:`echo $USERNAME` /home/`echo $USERNAME`" \ No newline at end of file diff --git a/exercise-materials/access-control-exercises/aws-example2.csv b/exercise-materials/access-control-exercises/aws-example2.csv new file mode 100644 index 00000000..7981302a --- /dev/null +++ b/exercise-materials/access-control-exercises/aws-example2.csv @@ -0,0 +1,4 @@ +wsi_series,wsi_issuer,wsi_issue_number,wsi_local,wmo_block_number,wmo_station_number,station_type,year,month,day,hour,minute,latitude,longitude,station_height_above_msl,barometer_height_above_msl,station_pressure,msl_pressure,geopotential_height,thermometer_height,air_temperature,dewpoint_temperature,relative_humidity,method_of_ground_state_measurement,ground_state,method_of_snow_depth_measurement,snow_depth,precipitation_intensity,anemometer_height,time_period_of_wind,wind_direction,wind_speed,maximum_wind_gust_direction_10_minutes,maximum_wind_gust_speed_10_minutes,maximum_wind_gust_direction_1_hour,maximum_wind_gust_speed_1_hour,maximum_wind_gust_direction_3_hours,maximum_wind_gust_speed_3_hours,rain_sensor_height,total_precipitation_1_hour,total_precipitation_3_hours,total_precipitation_6_hours,total_precipitation_12_hours,total_precipitation_24_hours +0,20000,0,60351,15,15,1,2024,3,31,1,0,47.77706163,23.94046026,503,,100940,101040,1448,5,298.15,294.55,80,3,1,1,0,0.004,10,-10,30,3,30,5,40,9,20,11,2,4.7,5.3,7.9,9.5,11.4 +0,20000,0,60355,15,15,1,2024,3,31,1,0,47.77706163,23.94046026,503,504.43,100940,101040,1448,5,298.15,294.55,80,3,1,1,0,0.004,10,-10,30,3,30,5,40,9,20,11,2,4.7,5.3,7.9,9.5,11.4 +0,20000,0,60360,15,15,1,2024,3,31,1,0,47.77706163,23.94046026,503,504.43,100940,101040,1448,5,298.15,294.55,80,3,1,1,0,0.004,10,-10,30,3,30,5,40,9,20,11,2,4.7,5.3,7.9,9.5,11.4 \ No newline at end of file diff --git a/exercise-materials/accessing-your-student-vm/hello_world.txt b/exercise-materials/accessing-your-student-vm/hello_world.txt deleted file mode 100644 index 916ea32a..00000000 --- a/exercise-materials/accessing-your-student-vm/hello_world.txt +++ /dev/null @@ -1 +0,0 @@ -Hello from your student-VM ! \ No newline at end of file diff --git a/exercise-materials/data-ingest/bufr-example.bin b/exercise-materials/data-ingest-exercises/bufr-example.bin similarity index 100% rename from exercise-materials/data-ingest/bufr-example.bin rename to exercise-materials/data-ingest-exercises/bufr-example.bin diff --git a/exercise-materials/data-ingest/copy_data_to_incoming.py b/exercise-materials/data-ingest-exercises/copy_file_to_incoming.py similarity index 66% rename from exercise-materials/data-ingest/copy_data_to_incoming.py rename to exercise-materials/data-ingest-exercises/copy_file_to_incoming.py index 4d7cf9fc..42cb1e42 100644 --- a/exercise-materials/data-ingest/copy_data_to_incoming.py +++ b/exercise-materials/data-ingest-exercises/copy_file_to_incoming.py @@ -1,15 +1,17 @@ from minio import Minio -# replace with location of your data -local_file = 'bufr-example.bin' -# define path tp match desired topic-hierarchy for your data -minio_path = 'xyz/test/data/core/weather/surface-based-observations/synop' +# define the local file path from the first argument provided to the script +import sys +local_file = sys.argv[1] + +# the path in the MinIO storage, matching dataset-id or topic +minio_path = 'urn:wmo:md:nl-knmi-test:core.surface-based-observations.synop' # replace with your host STORAGE_ENDPOINT = 'http://:9000' # your MinIO storage credentials STORAGE_USER = 'wis2box' -STORAGE_PASSWORD = 'changetoyourpassword' +STORAGE_PASSWORD = 'mystoragepassword' BUCKET_INCOMING = 'wis2box-incoming' if STORAGE_ENDPOINT.startswith('https://'): @@ -24,6 +26,6 @@ access_key=STORAGE_USER, secret_key=STORAGE_PASSWORD, secure=is_secure) -identifier = minio_path+local_file.split('/')[-1] +identifier = minio_path+'/'+local_file.split('/')[-1] print(f"Put into {BUCKET_INCOMING} : {local_file} as {identifier}") client.fput_object(BUCKET_INCOMING, identifier, local_file) \ No newline at end of file diff --git a/exercise-materials/data-ingest-exercises/csv-aws-example.csv b/exercise-materials/data-ingest-exercises/csv-aws-example.csv new file mode 100644 index 00000000..5e81f3fa --- /dev/null +++ b/exercise-materials/data-ingest-exercises/csv-aws-example.csv @@ -0,0 +1,2 @@ +wsi_series,wsi_issuer,wsi_issue_number,wsi_local,wmo_block_number,wmo_station_number,station_type,year,month,day,hour,minute,latitude,longitude,station_height_above_msl,barometer_height_above_msl,station_pressure,msl_pressure,geopotential_height,thermometer_height,air_temperature,dewpoint_temperature,relative_humidity,method_of_ground_state_measurement,ground_state,method_of_snow_depth_measurement,snow_depth,precipitation_intensity,anemometer_height,time_period_of_wind,wind_direction,wind_speed,maximum_wind_gust_direction_10_minutes,maximum_wind_gust_speed_10_minutes,maximum_wind_gust_direction_1_hour,maximum_wind_gust_speed_1_hour,maximum_wind_gust_direction_3_hours,maximum_wind_gust_speed_3_hours,rain_sensor_height,total_precipitation_1_hour,total_precipitation_3_hours,total_precipitation_6_hours,total_precipitation_12_hours,total_precipitation_24_hours +0,20000,0,60355,15,15,1,2023,3,31,1,0,47.77706163,23.94046026,503,,100940,101040,1448,5,298.15,294.55,80,3,1,1,0,0.004,10,-10,30,3,30,5,40,9,20,11,2,4.7,5.3,7.9,9.5,11.4 \ No newline at end of file diff --git a/documentation/docs/sample-data/synop-202307.txt b/exercise-materials/data-ingest-exercises/synop-202307.txt similarity index 100% rename from documentation/docs/sample-data/synop-202307.txt rename to exercise-materials/data-ingest-exercises/synop-202307.txt diff --git a/documentation/docs/sample-data/synop-202308.txt b/exercise-materials/data-ingest-exercises/synop-202308.txt similarity index 100% rename from documentation/docs/sample-data/synop-202308.txt rename to exercise-materials/data-ingest-exercises/synop-202308.txt diff --git a/exercise-materials/data-ingest-exercises/synop-202309.txt b/exercise-materials/data-ingest-exercises/synop-202309.txt new file mode 100644 index 00000000..0def8b7a --- /dev/null +++ b/exercise-materials/data-ingest-exercises/synop-202309.txt @@ -0,0 +1,4 @@ +SICG20 FCBB 030900 +AAXX 03094 +60355 42460 71004 10285 20245 30113 40133 8493/ + 333 59005 83813 81930 87363 94966 95836= \ No newline at end of file diff --git a/exercise-materials/data-ingest-exercises/synop.txt b/exercise-materials/data-ingest-exercises/synop.txt new file mode 100644 index 00000000..0def8b7a --- /dev/null +++ b/exercise-materials/data-ingest-exercises/synop.txt @@ -0,0 +1,4 @@ +SICG20 FCBB 030900 +AAXX 03094 +60355 42460 71004 10285 20245 30113 40133 8493/ + 333 59005 83813 81930 87363 94966 95836= \ No newline at end of file diff --git a/exercise-materials/data-ingest-exercises/wis2box-ftp.env b/exercise-materials/data-ingest-exercises/wis2box-ftp.env new file mode 100644 index 00000000..4059bad5 --- /dev/null +++ b/exercise-materials/data-ingest-exercises/wis2box-ftp.env @@ -0,0 +1,9 @@ +FTP_USER=wis2box +FTP_PASS=myftppassword +FTP_HOST=username.wis2.training + +WIS2BOX_STORAGE_ENDPOINT=http://minio:9000 +WIS2BOX_STORAGE_USERNAME=wis2box +WIS2BOX_STORAGE_PASSWORD=mywis2boxstoragepassword + +LOGGING_LEVEL=WARNING \ No newline at end of file diff --git a/exercise-materials/gts-headers-exercises/A_SMRO01YRBK171200_C_EDZW_20240717120502.txt b/exercise-materials/gts-headers-exercises/A_SMRO01YRBK171200_C_EDZW_20240717120502.txt new file mode 100644 index 00000000..5a664c2c --- /dev/null +++ b/exercise-materials/gts-headers-exercises/A_SMRO01YRBK171200_C_EDZW_20240717120502.txt @@ -0,0 +1,15 @@ +SMRO01 YRBK 171200 + +AAXX 17121 + +60355 01597 71702 10057 20036 39390 42628 50004 60021 78082 87300 333 + +4/000 + +55304 0//// 20643 3//// 69977 91003 91108= + +60360 02597 61303 10104 20040 39783 49976 58007 60001 83570 333 4/000 + +55308 + +0//// 21085 3//// 60007 91005 91106= \ No newline at end of file diff --git a/exercise-materials/gts-headers-exercises/gts_headers_mapping.csv b/exercise-materials/gts-headers-exercises/gts_headers_mapping.csv new file mode 100644 index 00000000..7ac0c670 --- /dev/null +++ b/exercise-materials/gts-headers-exercises/gts_headers_mapping.csv @@ -0,0 +1,3 @@ +string_in_filepath,ttaaii,cccc +SMRO01YRBK,SMRO01,YRBK +SMRO02YRBK,SMRO02,YRBK \ No newline at end of file diff --git a/exercise-materials/monitoring-exercises/aws-example.csv b/exercise-materials/monitoring-exercises/aws-example.csv new file mode 100644 index 00000000..66a18af9 --- /dev/null +++ b/exercise-materials/monitoring-exercises/aws-example.csv @@ -0,0 +1,7 @@ +wsi_series,wsi_issuer,wsi_issue_number,wsi_local,wmo_block_number,wmo_station_number,station_type,year,month,day,hour,minute,latitude,longitude,station_height_above_msl,barometer_height_above_msl,station_pressure,msl_pressure,geopotential_height,thermometer_height,air_temperature,dewpoint_temperature,relative_humidity,method_of_ground_state_measurement,ground_state,method_of_snow_depth_measurement,snow_depth,precipitation_intensity,anemometer_height,time_period_of_wind,wind_direction,wind_speed,maximum_wind_gust_direction_10_minutes,maximum_wind_gust_speed_10_minutes,maximum_wind_gust_direction_1_hour,maximum_wind_gust_speed_1_hour,maximum_wind_gust_direction_3_hours,maximum_wind_gust_speed_3_hours,rain_sensor_height,total_precipitation_1_hour,total_precipitation_3_hours,total_precipitation_6_hours,total_precipitation_12_hours,total_precipitation_24_hours +0,20000,0,60355,15,15,1,2024,3,31,1,0,47.77706163,23.94046026,503,504.43,100940,101040,1448,5,298.15,294.55,80,3,1,1,0,0.004,10,-10,30,3,30,5,40,9,20,11,2,4.7,5.3,7.9,9.5,11.4 +0,20000,0,60355,15,15,1,2024,3,31,2,0,47.77706163,23.94046026,503,504.43,100940,101040,1448,5,298.15,294.55,80,3,1,1,0,0.004,10,-10,30,3,30,5,40,9,20,11,2,4.7,5.3,7.9,9.5,11.4 +0,20000,0,60355,15,15,1,2024,3,31,3,0,47.77706163,23.94046026,503,504.43,100940,101040,1448,5,298.15,294.55,80,3,1,1,0,0.004,10,-10,30,3,30,5,40,9,20,11,2,4.7,5.3,7.9,9.5,11.4 +0,20000,0,60355,15,15,1,2024,3,31,4,0,47.77706163,23.94046026,503,504.43,100940,101040,1448,5,298.15,294.55,80,3,1,1,0,0.004,10,-10,30,3,30,5,40,9,20,11,2,4.7,5.3,7.9,9.5,11.4 +0,20000,0,60355,15,15,1,2024,3,31,5,0,47.77706163,23.94046026,503,504.43,100940,101040,1448,5,298.15,294.55,80,3,1,1,0,0.004,10,-10,30,3,30,5,40,9,20,11,2,4.7,5.3,7.9,9.5,11.4 +0,20000,0,60355,15,15,1,2024,3,31,6,0,47.77706163,23.94046026,503,504.43,100940,101040,1448,5,298.15,294.55,80,3,1,1,0,0.004,10,-10,30,3,30,5,40,9,20,11,2,4.7,5.3,7.9,9.5,11.4 diff --git a/exercise-materials/pywis-pubsub-exercises/config.yml b/exercise-materials/pywis-pubsub-exercises/config.yml deleted file mode 100644 index ae83a5a6..00000000 --- a/exercise-materials/pywis-pubsub-exercises/config.yml +++ /dev/null @@ -1,18 +0,0 @@ -broker: mqtt://username:password@host:port - -# list of topics to subscribe to -subscribe_topics: - - TBD - - TBD - -# whether to verify data -verify_data: true - -# whether to validate message against schema -validate_message: false - -# storage: filesystem -storage: - type: fs - options: - path: TBD diff --git a/exercise-materials/synop2bufr-exercises/ex_1/message.txt b/exercise-materials/synop2bufr-exercises/ex_1/message.txt new file mode 100644 index 00000000..76ffc3d2 --- /dev/null +++ b/exercise-materials/synop2bufr-exercises/ex_1/message.txt @@ -0,0 +1,2 @@ +AAXX 21121 +15015 02999 02501 10103 21090 39765 42952 57020 60001= \ No newline at end of file diff --git a/exercise-materials/synop2bufr-exercises/ex_1/station_list.csv b/exercise-materials/synop2bufr-exercises/ex_1/station_list.csv new file mode 100644 index 00000000..00234079 --- /dev/null +++ b/exercise-materials/synop2bufr-exercises/ex_1/station_list.csv @@ -0,0 +1,2 @@ +station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name,wmo_region +OCNA SUGATAG,0-20000-0-15015,15015,landFixed,47.7770616258,23.9404602638,503.0,504.0,ROU,europe \ No newline at end of file diff --git a/exercise-materials/synop2bufr-exercises/ex_2/message.txt b/exercise-materials/synop2bufr-exercises/ex_2/message.txt new file mode 100644 index 00000000..a497df88 --- /dev/null +++ b/exercise-materials/synop2bufr-exercises/ex_2/message.txt @@ -0,0 +1,4 @@ +AAXX 21121 +15015 02999 02501 10103 21090 39765 42952 57020 60001= +15020 02997 23104 10130 21075 30177 40377 58020 60001 81041= +15090 02997 53102 10139 21075 30271 40364 58031 60001 82046= \ No newline at end of file diff --git a/exercise-materials/synop2bufr-exercises/ex_2/station_list.csv b/exercise-materials/synop2bufr-exercises/ex_2/station_list.csv new file mode 100644 index 00000000..14feade2 --- /dev/null +++ b/exercise-materials/synop2bufr-exercises/ex_2/station_list.csv @@ -0,0 +1,4 @@ +station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name,wmo_region +OCNA SUGATAG,0-20000-0-15015,15015,landFixed,47.7770616258,23.9404602638,503.0,504.0,ROU,europe +BOTOSANI,0-20000-0-15020,15020,landFixed,47.7356532437,26.6455501701,161.0,162.1,ROU,europe +IASI,0-20000-0-15090,15090,landFixed,47.163333333,27.6272222222,74.29,72.39,ROU,europe \ No newline at end of file diff --git a/exercise-materials/synop2bufr-exercises/ex_3/message.txt b/exercise-materials/synop2bufr-exercises/ex_3/message.txt new file mode 100644 index 00000000..aa25e363 --- /dev/null +++ b/exercise-materials/synop2bufr-exercises/ex_3/message.txt @@ -0,0 +1,5 @@ +AAXX 21121 +15260 05515 32931 10103 21090 39765 42250 57020 60071 72006 82110 91155 +333 10178 21073 34101 55055 00010 20003 30002 50001 60004 +60035 70500 83145 81533 91008 91111 +444 18031 22053= \ No newline at end of file diff --git a/exercise-materials/synop2bufr-exercises/ex_3/station_list.csv b/exercise-materials/synop2bufr-exercises/ex_3/station_list.csv new file mode 100644 index 00000000..8fe007d2 --- /dev/null +++ b/exercise-materials/synop2bufr-exercises/ex_3/station_list.csv @@ -0,0 +1,9 @@ +station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name,wmo_region +OCNA SUGATAG,0-20000-0-15015,15015,landFixed,47.7770616258,23.9404602638,503.0,504.0,ROU,europe +BOTOSANI,0-20000-0-15020,15020,landFixed,47.7356532437,26.6455501701,161.0,162.1,ROU,europe +IASI,0-20000-0-15090,15090,landFixed,47.163333333,27.6272222222,74.29,72.39,ROU,europe +CEAHLAU TOACA,0-20000-0-15108,15108,landFixed,46.9775099973,25.9499399749,1897,1899.0,ROU,europe +CLUJ-NAPOCA,0-20000-0-15120,15120,landFixed,46.7777705044,23.5713052939,410,411.2,ROU,europe +BACAU,0-20000-0-15150,15150,landFixed,46.5577777778,26.8966666667,174,174.9,ROU,europe +MIERCUREA CIUC,0-20000-0-15170,15170,landFixed,46.3713166568,25.7726166755,661,662.3,ROU,europe +SIBIU,0-20000-0-15260,15260,landFixed,45.79018,24.036245,450,450.9,ROU,europe \ No newline at end of file diff --git a/exercise-materials/synop2bufr-exercises/ex_4/message_incorrect.txt b/exercise-materials/synop2bufr-exercises/ex_4/message_incorrect.txt new file mode 100644 index 00000000..efdc8eb7 --- /dev/null +++ b/exercise-materials/synop2bufr-exercises/ex_4/message_incorrect.txt @@ -0,0 +1,7 @@ +AAXX 21121 + +15015 02999 02501 10103 21090 39765 42952 57020 60001 + +15020 02997 23104 10130 21075 30177 40377 58020 60001 81041= + +15090 02997 53102 10139 21075 30271 40364 58031 60001 82046 \ No newline at end of file diff --git a/exercise-materials/synop2bufr-exercises/ex_4/station_list.csv b/exercise-materials/synop2bufr-exercises/ex_4/station_list.csv new file mode 100644 index 00000000..14feade2 --- /dev/null +++ b/exercise-materials/synop2bufr-exercises/ex_4/station_list.csv @@ -0,0 +1,4 @@ +station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name,wmo_region +OCNA SUGATAG,0-20000-0-15015,15015,landFixed,47.7770616258,23.9404602638,503.0,504.0,ROU,europe +BOTOSANI,0-20000-0-15020,15020,landFixed,47.7356532437,26.6455501701,161.0,162.1,ROU,europe +IASI,0-20000-0-15090,15090,landFixed,47.163333333,27.6272222222,74.29,72.39,ROU,europe \ No newline at end of file diff --git a/exercise-materials/synop2bufr-exercises/ex_5/message.txt b/exercise-materials/synop2bufr-exercises/ex_5/message.txt new file mode 100644 index 00000000..a497df88 --- /dev/null +++ b/exercise-materials/synop2bufr-exercises/ex_5/message.txt @@ -0,0 +1,4 @@ +AAXX 21121 +15015 02999 02501 10103 21090 39765 42952 57020 60001= +15020 02997 23104 10130 21075 30177 40377 58020 60001 81041= +15090 02997 53102 10139 21075 30271 40364 58031 60001 82046= \ No newline at end of file diff --git a/exercise-materials/synop2bufr-exercises/ex_5/station_list_incorrect.csv b/exercise-materials/synop2bufr-exercises/ex_5/station_list_incorrect.csv new file mode 100644 index 00000000..9afd2da5 --- /dev/null +++ b/exercise-materials/synop2bufr-exercises/ex_5/station_list_incorrect.csv @@ -0,0 +1,3 @@ +station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name,wmo_region +BOTOSANI,0-20000-0-15020,15020,landFixed,47.7356532437,26.6455501701,161.0,162.1,ROU,europe +IASI,0-20000-0-15090,15090,landFixed,47.163333333,27.6272222222,74.29,72.39,ROU,europe \ No newline at end of file