From 0a87a62ac52fb010c76756111f679b900a67db5b Mon Sep 17 00:00:00 2001 From: Maaike Date: Fri, 24 Nov 2023 14:43:26 +0100 Subject: [PATCH 1/2] remove country id --- README.md | 11 +++++------ test/docker-compose.data-subscriber.yml | 1 - test/run-tests.py | 2 +- wis2box-data-subscriber.py | 8 ++------ 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index dced76d..8b71d31 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # wis2box-data-subscriber -Service to subscribe to data published by Automatic Weather Stations and store the data into the wis2box-incoming bucket. It is currently designed to in conjuction with the 'csv2bufr'-plugin and the default mapping-template 'synop_bufr.json' . +Service to subscribe to data published by Automatic Weather Stations and store the data into the wis2box-incoming bucket. It is currently designed to work in conjunction with the 'csv2bufr'-plugin and the default mapping-template 'synop_bufr.json' and is compatible with Campbell scientific data loggers version CR1000X. ## how it works @@ -12,9 +12,9 @@ One .csv is created per timestamp for the content of the 'observations'-object i The .csv is stored as new object into the MINIO-endpoint defined by the environment variables for this services. The path used for the file in the minio-bucket is defined as: -`{COUNTRY_ID}/{CENTRE_ID}/data/core/weather/surface-based-observations/synop/` +`{CENTRE_ID}/data/core/weather/surface-based-observations/synop/` -Where COUNTRY_ID and CENTRE_ID are defined with environment variables. +Where CENTRE_ID is defined with an environment variables. Processing non-synop data will required an updated version of the current code. @@ -27,13 +27,12 @@ wis2box-data-subscriber uses. the following environment-variables: ```bash AWS_BROKER=mqtt://wis2box:xxx@localhost:1883 # set endpoint for broker where AWS data is published -COUNTRY_ID=zmb # set country_id used in wis2-topic-hierarchy -CENTRE_ID=zmb_met_centre # set centre_id for wis2-topic-hierarchy +CENTRE_ID=zm-zmb_met_centre # set centre_id for wis2-topic-hierarchy LOGGING_LEVEL=INFO # set logging level LOGGING_LEVEL=INFO # set logging-level MINIO_BUCKET=wis2box-incoming MINIO_ENDPOINT=http://localhost:9000 # set this to the minio-endpoint for your wis2box -MINIO_ROOT_USER=minio # minio username for your minio-endpoint +MINIO_ROOT_USER=wis2box # minio username for your minio-endpoint MINIO_ROOT_PASSWORD=xxx # minio password for your minio-endpoint ``` diff --git a/test/docker-compose.data-subscriber.yml b/test/docker-compose.data-subscriber.yml index 4eb05a5..422d5a3 100644 --- a/test/docker-compose.data-subscriber.yml +++ b/test/docker-compose.data-subscriber.yml @@ -5,7 +5,6 @@ services: environment: - LOGGING_LEVEL=${LOGGING_LEVEL:-WARNING} - CENTRE_ID=${CENTRE_ID} - - COUNTRY_ID=${COUNTRY_ID} - AWS_BROKER=${AWS_BROKER} - MINIO_BUCKET=${MINIO_BUCKET} - MINIO_ENDPOINT=${MINIO_ENDPOINT} diff --git a/test/run-tests.py b/test/run-tests.py index 9b025b3..04c8cdb 100644 --- a/test/run-tests.py +++ b/test/run-tests.py @@ -22,7 +22,7 @@ print('wait 2 seconds') time.sleep(2) print('check file is in bucket') -mypath = 'zmb/zmb_met_centre/data/core/weather/surface-based-observations/synop' +mypath = 'zm-zmb_met_centre/data/core/weather/surface-based-observations/synop' myfilename = 'CR1000X_34769_20230428T153200Z.csv' res = requests.get(f'http://localhost:9000/wis2box-incoming/{mypath}/{myfilename}') print(res) diff --git a/wis2box-data-subscriber.py b/wis2box-data-subscriber.py index d54a736..ec4758f 100644 --- a/wis2box-data-subscriber.py +++ b/wis2box-data-subscriber.py @@ -37,7 +37,6 @@ LOGGING_LOGLEVEL = os.environ.get('LOGGING_LEVEL', 'INFO') CENTRE_ID = os.environ.get('CENTRE_ID', 'not defined') -COUNTRY_ID = os.environ.get('COUNTRY_ID', 'not defined') AWS_BROKER = os.environ.get('AWS_BROKER', 'not defined') MINIO_ENDPOINT = os.environ.get('MINIO_ENDPOINT') @@ -94,7 +93,7 @@ def write_csv(topic, headers, logger_id, data): base_csv += f'"{logger_id}"\n' base_csv += '"observations:"\n' - prefix = f"{COUNTRY_ID}/{CENTRE_ID}" + prefix = f"{CENTRE_ID}" prefix += "/data/core/weather/surface-based-observations/synop/" # create one csv per timestamp @@ -181,9 +180,6 @@ def main(): print(f"LOGGING_LOGLEVEL={LOGGING_LOGLEVEL}") LOGGER.setLevel(LOGGING_LOGLEVEL) - if COUNTRY_ID == 'not defined': - LOGGER.error("COUNTRY_ID is not defined, exiting") - return if CENTRE_ID == 'not defined': LOGGER.error("CENTRE_ID is not defined, exiting") return @@ -198,7 +194,7 @@ def main(): LOGGER.info(f"host={broker_url.hostname}") LOGGER.info(f"user={broker_url.username}") LOGGER.info(f"port={broker_url.port}") - client_id = f"wis2box-data-subscriber-{COUNTRY_ID}-{CENTRE_ID}" + client_id = f"wis2box-data-subscriber-{CENTRE_ID}" client = mqtt.Client(client_id=client_id, protocol=mqtt.MQTTv5) client.on_connect = sub_data_incoming client.on_message = process_data From cd50179da9eda981f2ead6a14d958447f1b7c6c8 Mon Sep 17 00:00:00 2001 From: Maaike Date: Fri, 24 Nov 2023 15:53:30 +0100 Subject: [PATCH 2/2] update env_file for test --- test/env_file | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/env_file b/test/env_file index 29779dc..72d271c 100644 --- a/test/env_file +++ b/test/env_file @@ -1,6 +1,5 @@ AWS_BROKER=mqtt://wis2box@localhost:5883 -COUNTRY_ID=zmb -CENTRE_ID=zmb_met_centre +CENTRE_ID=zm-zmb_met_centre LOGGING_LEVEL=INFO MINIO_BUCKET=wis2box-incoming