From 21cce4e14dd7ebf4491be4f159b66bb9dc6397f7 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 2 Jan 2018 09:15:35 +0100 Subject: [PATCH 01/37] Add hostname in "healthy again" hipchat message --- dcompose-stack/radar-cp-hadoop-stack/check-health.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/check-health.sh b/dcompose-stack/radar-cp-hadoop-stack/check-health.sh index 43e7156d3..fb37b632b 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/check-health.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/check-health.sh @@ -45,10 +45,12 @@ while read service; do fi done <<< "$(sudo-linux docker-compose config --services)" +display_host="${SERVER_NAME} ($(hostname -f), $(curl -s http://ipecho.net/plain))" + if [ "${#unhealthy[@]}" -eq 0 ]; then if [ -f .unhealthy ]; then rm -f .unhealthy - hipchat_notify green "All services are healthy again" + hipchat_notify green "All services on ${display_host} are healthy again" fi echo "All services are healthy" else @@ -57,14 +59,15 @@ else # Send notification to MAINTAINER # start up the mail container if not already started sudo-linux docker-compose up -d smtp + # ensure that all topics are available + sudo-linux docker-compose run --rm kafka-init # save the container, so that we can use exec to send an email later container=$(sudo-linux docker-compose ps -q smtp) SAVEIFS=$IFS IFS=, display_services="[${unhealthy[*]}]" IFS=$SAVEIFS - display_host="${SERVER_NAME} ($(hostname -f), $(curl -s http://ipecho.net/plain))" - body="Services on $display_host are unhealthy. Services $display_services have been restarted. Please log in for further information." + body="Services on ${display_host} are unhealthy. Services $display_services have been restarted. Please log in for further information." echo "Sent notification to $MAINTAINER_EMAIL" echo "$body" | sudo-linux docker exec -i ${container} mail -aFrom:$FROM_EMAIL "-s[RADAR] Services on ${SERVER_NAME} unhealthy" $MAINTAINER_EMAIL From 1609081f54f70e6ddde6fbaf81469fe8a85c66c3 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Thu, 1 Feb 2018 16:25:16 +0000 Subject: [PATCH 02/37] Update image versions and add postgres backup script --- .idea/RADAR-Docker.iml | 9 + .idea/modules.xml | 8 + .idea/vcs.xml | 6 + .idea/workspace.xml | 616 ++++++++++++++++++ .../radar-cp-hadoop-stack/docker-compose.yml | 12 +- .../kafka-radarinit/Dockerfile | 4 +- .../postgres-backup/README.md | 16 + .../postgres-backup/scripts/pg_backup.config | 45 ++ .../scripts/pg_backup_rotated.sh | 201 ++++++ images/radar-backend-kafka/Dockerfile | 4 +- images/radar-hdfs-connector/Dockerfile | 2 +- images/radar-mongodb-connector/Dockerfile | 2 +- images/radar-restapi/Dockerfile | 2 +- 13 files changed, 915 insertions(+), 12 deletions(-) create mode 100644 .idea/RADAR-Docker.iml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml create mode 100644 dcompose-stack/radar-cp-hadoop-stack/postgres-backup/README.md create mode 100644 dcompose-stack/radar-cp-hadoop-stack/postgres-backup/scripts/pg_backup.config create mode 100644 dcompose-stack/radar-cp-hadoop-stack/postgres-backup/scripts/pg_backup_rotated.sh diff --git a/.idea/RADAR-Docker.iml b/.idea/RADAR-Docker.iml new file mode 100644 index 000000000..d6ebd4805 --- /dev/null +++ b/.idea/RADAR-Docker.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 000000000..1aef5bf6e --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 000000000..35eb1ddfb --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 000000000..26ab3d446 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,616 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + DEFINITION_ORDER + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1517492277293 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1.8 - - - - - - - - 1.8 - - - - - - - - - - - - - - - \ No newline at end of file From 7db6cf10a045b04f3edbcfff140e36368668f976 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Thu, 1 Feb 2018 16:34:22 +0000 Subject: [PATCH 04/37] Use relative path --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 5cac815e5..9af94d6ce 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -606,8 +606,8 @@ services: image: postgres:9.6.2 volumes: - "${MP_POSTGRES_DIR}/data/:/var/lib/postgresql/data/" - - "postgres-backup/backups/postgresql:/backups/database/postgresql/" - - "postgres-backup/scripts:/backup-scripts" + - "./postgres-backup/backups/postgresql:/backups/database/postgresql/" + - "./postgres-backup/scripts:/backup-scripts" environment: POSTGRES_USER : ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} From a5faa2742bb6912bb7d84f68868788ae0d702c65 Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Thu, 1 Feb 2018 17:15:33 +0000 Subject: [PATCH 05/37] Use MP 0.3.4 --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 9af94d6ce..b051afe46 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -570,7 +570,7 @@ services: # Management Portal # #---------------------------------------------------------------------------# managementportal-app: - image: radarcns/management-portal:0.3.3 + image: radarcns/management-portal:0.3.4 networks: - default - api From 1d4298dbe6915e4de5de9bbc425742b5c5cfa0c5 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Fri, 2 Feb 2018 10:43:59 +0000 Subject: [PATCH 06/37] Further rename of RADAR-CNS to base --- README.md | 24 +++++++++---------- .../radar-cp-hadoop-stack/README.md | 4 ++-- .../radar-cp-hadoop-stack/hdfs_restructure.sh | 2 +- .../install-radar-stack.sh | 2 +- .../kafka-radarinit/topic_init.sh | 6 ++--- .../reboot-radar-stack.sh | 2 +- .../start-radar-stack.sh | 2 +- .../radar-cp-hadoop-stack/stop-radar-stack.sh | 2 +- dcompose-stack/radar-cp-hadoop-stack/util.sh | 2 +- images/radar-backend-kafka/Dockerfile | 2 +- images/radar-backend-kafka/README.md | 4 ++-- images/radar-backend-kafka/init.sh | 2 +- images/radar-hdfs-connector/Dockerfile | 2 +- images/radar-hdfs-connector/README.md | 4 ++-- images/radar-hotstorage-mongodb/db_init.sh | 6 ++--- images/radar-mongodb-connector/Dockerfile | 2 +- images/radar-mongodb-connector/README.md | 4 ++-- images/radar-restapi/Dockerfile | 2 +- scripts/README.md | 2 +- wip/radar-cp-swarm-stack/README.md | 2 +- 20 files changed, 39 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index a3ba7a5bd..3a7e2014c 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # RADAR-Docker -The dockerized RADAR stack for deploying the RADAR-CNS platform. Component repositories can be found at [RADAR-CNS DockerHub org](https://hub.docker.com/u/radarcns/dashboard/) +The dockerized RADAR stack for deploying the RADAR-base platform. Component repositories can be found at [RADAR-base DockerHub org](https://hub.docker.com/u/radarcns/dashboard/) ## Installation instructions -To install RADAR-CNS stack, do the following: +To install RADAR-base stack, do the following: 1. Install [Docker Engine](https://docs.docker.com/engine/installation/) -2. Install `docker-compose` using the [installation guide](https://docs.docker.com/compose/install/) or by following our [wiki](https://github.com/RADAR-CNS/RADAR-Docker/wiki/How-to-set-up-docker-on-ubuntu#install-docker-compose). +2. Install `docker-compose` using the [installation guide](https://docs.docker.com/compose/install/) or by following our [wiki](https://github.com/RADAR-base/RADAR-Docker/wiki/How-to-set-up-docker-on-ubuntu#install-docker-compose). 3. Verify the Docker installation by running on the command-line: ```shell @@ -21,10 +21,10 @@ To install RADAR-CNS stack, do the following: sudo apt-get install git ``` -5. Clone [RADAR-Docker](https://github.com/RADAR-CNS/RADAR-Docker) repository from GitHub. +5. Clone [RADAR-Docker](https://github.com/RADAR-base/RADAR-Docker) repository from GitHub. ```shell - git clone https://github.com/RADAR-CNS/RADAR-Docker.git + git clone https://github.com/RADAR-base/RADAR-Docker.git ``` 6. Install required component stack following the instructions below. @@ -34,7 +34,7 @@ To install RADAR-CNS stack, do the following: RADAR-Docker currently offers two component stacks to run. 1. A Docker-compose for components from [Confluent Kafka Platform](http://docs.confluent.io/3.1.0/) community -2. A Docker-compose for components from RADAR-CNS platform. +2. A Docker-compose for components from RADAR-base platform. > **Note**: on macOS, remove `sudo` from all `docker` and `docker-compose` commands in the usage instructions below. @@ -54,23 +54,23 @@ To stop this stack, run: sudo docker-compose down ``` -### RADAR-CNS platform +### RADAR-base platform -In addition to Confluent Kafka platform components, RADAR-CNS platform offers +In addition to Confluent Kafka platform components, RADAR-base platform offers * RADAR-HDFS-Connector - Cold storage of selected streams in Hadoop data storage, * RADAR-MongoDB-Connector - Hot storage of selected streams in MongoDB, -* [RADAR-Dashboard](https://github.com/RADAR-CNS/RADAR-Dashboard), +* [RADAR-Dashboard](https://github.com/RADAR-base/RADAR-Dashboard), * RADAR-Streams - real-time aggregated streams, * RADAR-Monitor - Status monitors, -* [RADAR-HotStorage](https://github.com/RADAR-CNS/RADAR-HotStorage) via MongoDB, -* [RADAR-REST API](https://github.com/RADAR-CNS/RADAR-RestApi), +* [RADAR-HotStorage](https://github.com/RADAR-base/RADAR-HotStorage) via MongoDB, +* [RADAR-REST API](https://github.com/RADAR-base/RADAR-RestApi), * A Hadoop cluster, and * An email server. * Management Portal - A web portal to manage patient monitoring studies. * RADAR-Gateway - A validating gateway to allow only valid and authentic data to the platform * Catalog server - A Service to share source-types configured in the platform. -To run RADAR-CNS stack in a single node setup: +To run RADAR-base stack in a single node setup: 1. Navigate to `radar-cp-hadoop-stack`: diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index 2d9d46b61..cab469561 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -13,7 +13,7 @@ This docker-compose stack contains the full operational RADAR platform. Once con 2. Copy `etc/smtp.env.template` to `etc/smtp.env` and configure your email settings. Configure alternative mail providers like Amazon SES or Gmail by using the parameters of the [`namshi/smtp` Docker image](https://hub.docker.com/r/namshi/smtp/). -3. Copy `etc/redcap-integration/radar.yml.template` to `etc/redcap-integration/radar.yml` and modify it to configure the properties of Redcap instance and the management portal. For reference on configuration of this file look at the Readme file here - . In the REDcap portal under Project Setup, define the Data Trigger as `https:///redcapint/trigger` +3. Copy `etc/redcap-integration/radar.yml.template` to `etc/redcap-integration/radar.yml` and modify it to configure the properties of Redcap instance and the management portal. For reference on configuration of this file look at the Readme file here - . In the REDcap portal under Project Setup, define the Data Trigger as `https:///redcapint/trigger` 4. Copy `etc/managementportal/config/oauth_client_details.csv.template` to `etc/managementportal/config/oauth_client_details.csv` and change OAuth client credentials for production MP. The OAuth client for the frontend will be loaded automatically and does not need to be listed in this file. This file will be read at each startup. The current implementation overwrites existing clients with the same client ID, so be aware of this if you have made changes to a client listed in this file using the Management Portal frontend. This behaviour might change in the future. @@ -22,7 +22,7 @@ This docker-compose stack contains the full operational RADAR platform. Once con 6. (Optional) Note: To have different flush.size for different topics, you can create multipe property configurations for a single connector. To do that, 6.1 Create multipe property files that have different `flush.size` for given topics. - Examples [sink-hdfs-high.properties](https://github.com/RADAR-CNS/RADAR-Docker/blob/dev/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-high.properties) , [sink-hdfs-low.properties](https://github.com/RADAR-CNS/RADAR-Docker/blob/dev/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-low.properties) + Examples [sink-hdfs-high.properties](https://github.com/RADAR-base/RADAR-Docker/blob/dev/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-high.properties) , [sink-hdfs-low.properties](https://github.com/RADAR-base/RADAR-Docker/blob/dev/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-low.properties) 6.2 Add `CONNECTOR_PROPERTY_FILE_PREFIX: ` environment variable to `radar-hdfs-connector` service in `docker-compose` file. diff --git a/dcompose-stack/radar-cp-hadoop-stack/hdfs_restructure.sh b/dcompose-stack/radar-cp-hadoop-stack/hdfs_restructure.sh index bc3afebd6..a7a3c6a7f 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/hdfs_restructure.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/hdfs_restructure.sh @@ -14,7 +14,7 @@ JAR="restructurehdfs-all-${JAR_VERSION}.jar" if [ ! -e "lib/${JAR}" ]; then echo "Downloading HDFS restructuring JAR version ${JAR_VERSION}" - sudo-linux curl -L -# -o lib/${JAR} "https://github.com/RADAR-CNS/Restructure-HDFS-topic/releases/download/v${JAR_VERSION}/${JAR}" + sudo-linux curl -L -# -o lib/${JAR} "https://github.com/RADAR-base/Restructure-HDFS-topic/releases/download/v${JAR_VERSION}/${JAR}" fi # HDFS filename to get diff --git a/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh b/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh index 3fe872424..7b03188a0 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh @@ -114,7 +114,7 @@ inline_variable 'server_name[[:space:]]*' "${SERVER_NAME};" etc/webserver/nginx. sed_i 's|\(/etc/letsencrypt/live/\)[^/]*\(/.*\.pem\)|\1'"${SERVER_NAME}"'\2|' etc/webserver/nginx.conf init_certificate "${SERVER_NAME}" -echo "==> Starting RADAR-CNS Platform" +echo "==> Starting RADAR-base Platform" sudo-linux docker-compose up -d "$@" request_certificate "${SERVER_NAME}" "${SELF_SIGNED_CERT:-yes}" diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/topic_init.sh b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/topic_init.sh index 8b6b12e34..1c85cc8f0 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/topic_init.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/topic_init.sh @@ -1,7 +1,7 @@ #!/bin/bash # Create topics -echo "Creating RADAR-CNS topics..." +echo "Creating RADAR-base topics..." if ! radar-schemas-tools create -p $KAFKA_NUM_PARTITIONS -r $KAFKA_NUM_REPLICATION -b $KAFKA_NUM_BROKERS "${KAFKA_ZOOKEEPER_CONNECT}" merged; then echo "FAILED TO CREATE TOPICS" @@ -11,7 +11,7 @@ fi echo "Topics created." -echo "Registering RADAR-CNS schemas..." +echo "Registering RADAR-base schemas..." tries=10 timeout=1 @@ -40,5 +40,5 @@ fi echo "Schemas registered." echo "*******************************************" -echo "** RADAR-CNS topics and schemas ready **" +echo "** RADAR-base topics and schemas ready **" echo "*******************************************" diff --git a/dcompose-stack/radar-cp-hadoop-stack/reboot-radar-stack.sh b/dcompose-stack/radar-cp-hadoop-stack/reboot-radar-stack.sh index ae5f1df49..3b0e40e82 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/reboot-radar-stack.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/reboot-radar-stack.sh @@ -2,5 +2,5 @@ . ./util.sh -echo "==> Restarting RADAR-CNS Platform" +echo "==> Restarting RADAR-base Platform" sudo-linux docker-compose restart diff --git a/dcompose-stack/radar-cp-hadoop-stack/start-radar-stack.sh b/dcompose-stack/radar-cp-hadoop-stack/start-radar-stack.sh index a150c8e02..b44163f9d 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/start-radar-stack.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/start-radar-stack.sh @@ -2,5 +2,5 @@ . ./util.sh -echo "==> Starting radar-cns platform" +echo "==> Starting radar-base platform" sudo-linux docker-compose start diff --git a/dcompose-stack/radar-cp-hadoop-stack/stop-radar-stack.sh b/dcompose-stack/radar-cp-hadoop-stack/stop-radar-stack.sh index a11b6defa..8006f3fa0 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/stop-radar-stack.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/stop-radar-stack.sh @@ -2,5 +2,5 @@ . ./util.sh -echo "==> Stopping RADAR-CNS Stack" +echo "==> Stopping RADAR-base Stack" sudo-linux docker-compose stop diff --git a/dcompose-stack/radar-cp-hadoop-stack/util.sh b/dcompose-stack/radar-cp-hadoop-stack/util.sh index ec8ad0cd7..0a534d696 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/util.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/util.sh @@ -26,7 +26,7 @@ check_parent_exists() { fi PARENT=$(dirname $2) if [ ! -d "${PARENT}" ]; then - echo "RADAR-CNS stores volumes at ${PARENT}. If this folder does not exist, please create the entire path and then try again" + echo "RADAR-base stores volumes at ${PARENT}. If this folder does not exist, please create the entire path and then try again" exit 1 fi if [ -d "$2" ]; then diff --git a/images/radar-backend-kafka/Dockerfile b/images/radar-backend-kafka/Dockerfile index 42fe09ce5..8a101b64e 100644 --- a/images/radar-backend-kafka/Dockerfile +++ b/images/radar-backend-kafka/Dockerfile @@ -15,7 +15,7 @@ FROM confluentinc/cp-base:3.2.1 MAINTAINER Nivethika M , Joris B , Francesco N -LABEL description="RADAR-CNS Backend streams and monitor" +LABEL description="RADAR-base Backend streams and monitor" # Install RADAR-Backend RUN echo "==> Installing Components" \ diff --git a/images/radar-backend-kafka/README.md b/images/radar-backend-kafka/README.md index 5fca83d13..851573df5 100644 --- a/images/radar-backend-kafka/README.md +++ b/images/radar-backend-kafka/README.md @@ -1,6 +1,6 @@ # Dockerised RADAR-Backend-Kafka -It runs the RADAR-CNS Backend Kafka solution based on Kafka Streams 3.1.2, for more details about Kafka Streams click [here](http://docs.confluent.io/3.1.2/streams/index.html). +It runs the RADAR-base Backend Kafka solution based on Kafka Streams 3.1.2, for more details about Kafka Streams click [here](http://docs.confluent.io/3.1.2/streams/index.html). Create the docker image: ``` @@ -27,4 +27,4 @@ Before starting the streams, it waits until all topics inside TOPIC_LIST are ava ## How to run -For a complete use case scenario, check the RADAR-CNS `docker-compose` file available [here](https://github.com/RADAR-CNS/RADAR-Docker/blob/backend-integration/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml) +For a complete use case scenario, check the RADAR-base `docker-compose` file available [here](https://github.com/RADAR-base/RADAR-Docker/blob/backend-integration/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml) diff --git a/images/radar-backend-kafka/init.sh b/images/radar-backend-kafka/init.sh index 3f36a6835..faaccf9f1 100755 --- a/images/radar-backend-kafka/init.sh +++ b/images/radar-backend-kafka/init.sh @@ -1,7 +1,7 @@ #!/bin/bash # Busy waiting loop that waits untill all topic are available -echo "===> Waiting RADAR-CNS topics ... " +echo "===> Waiting RADAR-base topics ... " ./home/kafka_status.sh # Start streams diff --git a/images/radar-hdfs-connector/Dockerfile b/images/radar-hdfs-connector/Dockerfile index 2f61968e5..bffddce29 100644 --- a/images/radar-hdfs-connector/Dockerfile +++ b/images/radar-hdfs-connector/Dockerfile @@ -15,7 +15,7 @@ FROM confluentinc/cp-kafka-connect:3.2.1 MAINTAINER Nivethika M , Joris B -LABEL description="RADAR-CNS Backend- HDFS Sink Connector" +LABEL description="RADAR-base Backend- HDFS Sink Connector" # Deploy RADAR-Backend - HDFS Sink Connector ADD https://github.com/RADAR-base/RADAR-HDFS-Sink-Connector/releases/download/v0.1/radar-hdfs-sink-connector-0.1-sources.jar /etc/kafka-connect/jars/ diff --git a/images/radar-hdfs-connector/README.md b/images/radar-hdfs-connector/README.md index e055c859b..34feb3f8c 100644 --- a/images/radar-hdfs-connector/README.md +++ b/images/radar-hdfs-connector/README.md @@ -1,6 +1,6 @@ # Dockerised RADAR-HDFS-Connector -It runs the Confluent HDFS Connector 3.1.2 using a custom [RecordWriterProvider](https://github.com/RADAR-CNS/RADAR-Backend/blob/dev/src/main/java/org/radarcns/sink/hdfs/AvroRecordWriterProviderRadar.java) to support RADAR-CNS Avro schemas. For more details about Confluent HDFS Connector click [here](http://docs.confluent.io/3.1.2/connect/connect-hdfs/docs/index.html). +It runs the Confluent HDFS Connector 3.1.2 using a custom [RecordWriterProvider](https://github.com/RADAR-base/RADAR-Backend/blob/dev/src/main/java/org/radarcns/sink/hdfs/AvroRecordWriterProviderRadar.java) to support RADAR-base Avro schemas. For more details about Confluent HDFS Connector click [here](http://docs.confluent.io/3.1.2/connect/connect-hdfs/docs/index.html). Create the docker image: ``` @@ -41,4 +41,4 @@ Note that connector's REST_PORT must be different from the one used by Rest-Prox ## How to run -For a complete use case scenario, check the RADAR-CNS `docker-compose` file available [here](https://github.com/RADAR-CNS/RADAR-Docker/blob/backend-integration/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml) +For a complete use case scenario, check the RADAR-base `docker-compose` file available [here](https://github.com/RADAR-base/RADAR-Docker/blob/backend-integration/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml) diff --git a/images/radar-hotstorage-mongodb/db_init.sh b/images/radar-hotstorage-mongodb/db_init.sh index 19f8109f8..34780d77b 100644 --- a/images/radar-hotstorage-mongodb/db_init.sh +++ b/images/radar-hotstorage-mongodb/db_init.sh @@ -10,7 +10,7 @@ done if [ -f /data/db/.radar_hotstorage_set ]; then echo "**********************************************" - echo "** RADAR-CNS Hotstorage is up and running **" + echo "** RADAR-base Hotstorage is up and running **" echo "**********************************************" exit 0 fi @@ -32,7 +32,7 @@ fi echo "=> MongoDB is ready" -echo "=> Creating DB and User for RADAR-CNS Hot Storage" +echo "=> Creating DB and User for RADAR-base Hot Storage" mongo admin --eval 'db.createUser( { user: "'${RADAR_USER}'", pwd: "'${RADAR_PWD}'", roles: [ { role: "root", db: "admin" } ] } )' mongo admin -u $RADAR_USER -p $RADAR_PWD < , Joris B , Francesco N -LABEL description="RADAR-CNS Backend- MongoDB Sink Connector" +LABEL description="RADAR-base Backend- MongoDB Sink Connector" # Deploy RADAR-Backend - Mongodb Sink Connector ADD https://github.com/RADAR-base/RADAR-MongoDB-Sink-Connector/releases/download/0.1/radar-mongodb-sink-connector-0.1.jar /etc/kafka-connect/jars/ diff --git a/images/radar-mongodb-connector/README.md b/images/radar-mongodb-connector/README.md index 7dee0c24a..db7ee90db 100644 --- a/images/radar-mongodb-connector/README.md +++ b/images/radar-mongodb-connector/README.md @@ -1,6 +1,6 @@ # Dockerised RADAR-MongoDB-Connector -It runs the RADAR-CNS MongoDB Connector project based on Confluent Platform 3.1.2, for more details check the [repository](https://github.com/RADAR-CNS/RADAR-MongoDbConnector). +It runs the RADAR-base MongoDB Connector project based on Confluent Platform 3.1.2, for more details check the [repository](https://github.com/RADAR-base/RADAR-MongoDbConnector). Create the docker image: ``` @@ -58,4 +58,4 @@ Note that connector's REST_PORT must be different from the one used by Rest-Prox ## How to run -For a complete use case scenario, check the RADAR-CNS `docker-compose` file available [here](https://github.com/RADAR-CNS/RADAR-Docker/blob/backend-integration/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml) +For a complete use case scenario, check the RADAR-base `docker-compose` file available [here](https://github.com/RADAR-base/RADAR-Docker/blob/backend-integration/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml) diff --git a/images/radar-restapi/Dockerfile b/images/radar-restapi/Dockerfile index 7e7f2ffb3..567dea665 100644 --- a/images/radar-restapi/Dockerfile +++ b/images/radar-restapi/Dockerfile @@ -17,7 +17,7 @@ ENV JAVA_OPTS=-Djava.security.egd=file:/dev/urandom MAINTAINER @fnobilia, @afolarin -LABEL description="RADAR-CNS Rest Api docker container" +LABEL description="RADAR-base Rest Api docker container" # Install Rest API ADD https://github.com/RADAR-base/RADAR-RestApi/releases/download/v0.1.1/radar-0.1.1.war /usr/local/tomcat/webapps/radar.war diff --git a/scripts/README.md b/scripts/README.md index daf362a38..1aca271d3 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,6 +1,6 @@ ## Scripts -This folder contains useful scripts to manage the server where the RADAR-CNS Platform is running. +This folder contains useful scripts to manage the server where the RADAR-base Platform is running. - `check_radar_network.sh` checks if the machine is connected to internet. The script can be parametrised with - `nic` is the internet gateway diff --git a/wip/radar-cp-swarm-stack/README.md b/wip/radar-cp-swarm-stack/README.md index 5a9061220..ae78533dc 100644 --- a/wip/radar-cp-swarm-stack/README.md +++ b/wip/radar-cp-swarm-stack/README.md @@ -1,4 +1,4 @@ -# RADAR-CNS with multi-node cluster using Docker Swarm +# RADAR-base with multi-node cluster using Docker Swarm # Run the full setup with ```shell From c83a0d4adb7d0363c36faa9f38dbba40b1607396 Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Tue, 13 Feb 2018 13:42:05 +0000 Subject: [PATCH 07/37] Use radarbase org wherever possible --- .../radar-cp-hadoop-stack/docker-compose.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index b051afe46..392c8c3a5 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -210,7 +210,7 @@ services: # RADAR Hot Storage # #---------------------------------------------------------------------------# hotstorage: - image: radarcns/radar-hotstorage:0.1 + image: radarbase/radar-hotstorage:0.1 networks: - hotstorage volumes: @@ -231,7 +231,7 @@ services: # RADAR REST API # #---------------------------------------------------------------------------# rest-api: - image: radarcns/radar-restapi:0.1.1 + image: radarbase/radar-restapi:0.1.1 networks: - hotstorage - api @@ -251,7 +251,7 @@ services: # RADAR REDCap Integration # #---------------------------------------------------------------------------# radar-integration: - image: radarcns/radar-redcapintegration:latest + image: radarbase/radar-redcapintegration:latest networks: - api - default @@ -356,7 +356,7 @@ services: # RADAR mongo connector # #---------------------------------------------------------------------------# radar-mongodb-connector: - image: radarcns/radar-mongodb-connector-auto:0.1 + image: radarbase/radar-mongodb-connector-auto:0.1 restart: on-failure volumes: - ./etc/sink-mongo.properties:/etc/kafka-connect/sink.properties @@ -407,7 +407,7 @@ services: # RADAR HDFS connector # #---------------------------------------------------------------------------# radar-hdfs-connector: - image: radarcns/radar-hdfs-connector-auto:0.2.1 + image: radarbase/radar-hdfs-connector-auto:0.1 restart: on-failure volumes: - ./etc/sink-hdfs.properties:/etc/kafka-connect/sink-hdfs.properties @@ -457,7 +457,7 @@ services: # RADAR backend streams # #---------------------------------------------------------------------------# radar-backend-stream: - image: radarcns/radar-backend:0.2.0 + image: radarbase/radar-backend:0.2.0 command: - stream networks: @@ -484,7 +484,7 @@ services: # RADAR backend monitor # #---------------------------------------------------------------------------# radar-backend-monitor: - image: radarcns/radar-backend:0.2.0 + image: radarbase/radar-backend:0.2.0 command: monitor networks: - zookeeper @@ -570,7 +570,7 @@ services: # Management Portal # #---------------------------------------------------------------------------# managementportal-app: - image: radarcns/management-portal:0.3.4 + image: radarbase/management-portal:0.3.4 networks: - default - api @@ -624,7 +624,7 @@ services: # Kafka Manager # #---------------------------------------------------------------------------# kafka-manager: - image: radarcns/kafka-manager + image: radarbase/kafka-manager networks: - zookeeper - kafka From 903e094994b44e0413a7f12a00de9c96723a4ccb Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Thu, 15 Feb 2018 21:57:02 +0000 Subject: [PATCH 08/37] Update dockerhub link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3a7e2014c..962c7f7cb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # RADAR-Docker -The dockerized RADAR stack for deploying the RADAR-base platform. Component repositories can be found at [RADAR-base DockerHub org](https://hub.docker.com/u/radarcns/dashboard/) +The dockerized RADAR stack for deploying the RADAR-base platform. Component repositories can be found at [RADAR-base DockerHub org](https://hub.docker.com/u/radarbase/dashboard/) ## Installation instructions To install RADAR-base stack, do the following: From 53b96db79d850b89be7cdb4d8bd3c024db6de269 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Tue, 20 Feb 2018 15:23:50 +0000 Subject: [PATCH 09/37] Add links to scripts --- dcompose-stack/radar-cp-hadoop-stack/postgres-backup/README.md | 2 +- .../postgres-backup/scripts/pg_backup.config | 1 + .../postgres-backup/scripts/pg_backup_rotated.sh | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/postgres-backup/README.md b/dcompose-stack/radar-cp-hadoop-stack/postgres-backup/README.md index d58237195..f97246685 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/postgres-backup/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/postgres-backup/README.md @@ -1,6 +1,6 @@ # POSTGRES Backup Scripts -The `scripts` directory contains a script for running roatated backups of a running postgres instance. In this case these scripts are mounted to the postgres container (using bind mounts) and then run to create backups. +The `scripts` directory contains a script for running roatated backups of a running postgres instance. In this case these scripts are mounted to the postgres container (using bind mounts) and then run to create backups. These scripts are taken from the [postgres wiki](https://wiki.postgresql.org/wiki/Automated_Backup_on_Linux) The backups can be configured to create daily, weekly and monthly backups by configuring the `pg_backup.config` configuration file. Please note that if you change the `BACKUP_DIR` in the config file then make sure to change the bind mount in the postgres container in docker-compose.yml file as well. Further configuration information is contained in the `pg_backup.config` file. diff --git a/dcompose-stack/radar-cp-hadoop-stack/postgres-backup/scripts/pg_backup.config b/dcompose-stack/radar-cp-hadoop-stack/postgres-backup/scripts/pg_backup.config index b6dadd98f..b26cccec0 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/postgres-backup/scripts/pg_backup.config +++ b/dcompose-stack/radar-cp-hadoop-stack/postgres-backup/scripts/pg_backup.config @@ -1,6 +1,7 @@ ############################## ## POSTGRESQL BACKUP CONFIG ## ############################## +# Source - https://wiki.postgresql.org/wiki/Automated_Backup_on_Linux # Optional system user to run backups as. If the user the script is running as doesn't match this # the script terminates. Leave blank to skip check. diff --git a/dcompose-stack/radar-cp-hadoop-stack/postgres-backup/scripts/pg_backup_rotated.sh b/dcompose-stack/radar-cp-hadoop-stack/postgres-backup/scripts/pg_backup_rotated.sh index 142988e92..5a8882580 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/postgres-backup/scripts/pg_backup_rotated.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/postgres-backup/scripts/pg_backup_rotated.sh @@ -1,5 +1,7 @@ #!/bin/bash +# Source - https://wiki.postgresql.org/wiki/Automated_Backup_on_Linux + ########################### ####### LOAD CONFIG ####### ########################### From 6f90c989feae8a64f4bfa3ace7aba1738a4d59a3 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Thu, 22 Feb 2018 15:00:00 +0000 Subject: [PATCH 10/37] directory structure for backend radar.yml --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 4 ++-- .../etc/{ => radar-backend}/radar.yml.template | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename dcompose-stack/radar-cp-hadoop-stack/etc/{ => radar-backend}/radar.yml.template (100%) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 392c8c3a5..e960b4643 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -473,7 +473,7 @@ services: - schema-registry-1 - kafka-init volumes: - - ./etc/radar.yml:/etc/radar.yml + - ./etc/radar-backend/radar.yml:/etc/radar.yml restart: always environment: KAFKA_REST_PROXY: http://rest-proxy-1:8082 @@ -499,7 +499,7 @@ services: - kafka-init - smtp volumes: - - ./etc/radar.yml:/etc/radar.yml + - ./etc/radar-backend/radar.yml:/etc/radar.yml - radar-backend-monitor-disconnect-data:/var/lib/radar/data restart: always environment: diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/radar.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/etc/radar.yml.template rename to dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template From bab8b396c8f351b5ae0825aec041eaaaba2800f3 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Thu, 22 Feb 2018 15:52:14 +0000 Subject: [PATCH 11/37] fix install and travis scripts --- .travis.yml | 2 +- dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 42b2bec3d..86699fc7b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,7 +35,7 @@ script: - echo $"SMARTHOST_ADDRESS=mail.example.com\nSMARTHOST_PORT=587\nSMARTHOST_USER=user@example.com\nSMARTHOST_PASSWORD=XXXXXXXX" > etc/smtp.env - sudo docker volume create certs - sudo docker volume create certs-data - - cp etc/radar.yml.template etc/radar.yml + - cp etc/radar-backend/radar.yml.template etc/radar-backend/radar.yml - cp etc/webserver/nginx.conf.template etc/webserver/nginx.conf - cp etc/sink-hdfs.properties.template etc/sink-hdfs.properties - cp etc/sink-mongo.properties.template etc/sink-mongo.properties diff --git a/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh b/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh index 7b03188a0..10d6f7f2b 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh @@ -12,7 +12,7 @@ fi # Initialize and check all config files check_config_present .env etc/env.template -check_config_present etc/radar.yml +check_config_present etc/radar-backend/radar.yml check_config_present etc/managementportal/config/oauth_client_details.csv check_config_present etc/rest-api/mp_info.yml check_config_present etc/redcap-integration/radar.yml From 514ec1fcbe83e8802aa4e53681cad68ff588c929 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Thu, 22 Feb 2018 16:16:31 +0000 Subject: [PATCH 12/37] use latest backend and config --- .../radar-cp-hadoop-stack/docker-compose.yml | 4 ++-- .../etc/radar-backend/radar.yml.template | 20 +++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index e960b4643..0c84f7b17 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -457,7 +457,7 @@ services: # RADAR backend streams # #---------------------------------------------------------------------------# radar-backend-stream: - image: radarbase/radar-backend:0.2.0 + image: radarbase/radar-backend:0.2.1 command: - stream networks: @@ -484,7 +484,7 @@ services: # RADAR backend monitor # #---------------------------------------------------------------------------# radar-backend-monitor: - image: radarbase/radar-backend:0.2.0 + image: radarbase/radar-backend:0.2.1 command: monitor networks: - zookeeper diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template index 9bda09a38..459b1cfae 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template @@ -44,9 +44,13 @@ rest_proxy: #======================== Battery level monitor ========================# battery_monitor: level: LOW - email_address: - - notifier@email - - notifier2@email + notify: # Each project can have a number of email addresses + - project_id: s1 + email_address: + - notifier1@email + - project_id: s2 + email_address: + - notifier2@email email_host: localhost email_port: 25 email_user: user@example.com @@ -55,9 +59,13 @@ battery_monitor: #======================= Disconnection monitor==========================# disconnect_monitor: - email_address: - - notifier@email - - notifier2@email + notify: # Each project can have a number of email addresses + - project_id: s1 + email_address: + - notifier1@email + - project_id: s2 + email_address: + - notifier2@email email_host: localhost email_port: 25 email_user: user@example.com From fe80bf907b143aa0d59a3baba166491479dc9890 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Thu, 22 Feb 2018 17:02:39 +0000 Subject: [PATCH 13/37] remove radar-backend-kafka --- images/kafka-manager/README.md | 3 +- images/radar-backend-kafka/Dockerfile | 33 ------------ images/radar-backend-kafka/README.md | 30 ----------- images/radar-backend-kafka/init.sh | 9 ---- images/radar-backend-kafka/kafka_status.sh | 58 ---------------------- images/radar-backend-kafka/radar.yml | 37 -------------- 6 files changed, 1 insertion(+), 169 deletions(-) delete mode 100644 images/radar-backend-kafka/Dockerfile delete mode 100644 images/radar-backend-kafka/README.md delete mode 100755 images/radar-backend-kafka/init.sh delete mode 100755 images/radar-backend-kafka/kafka_status.sh delete mode 100644 images/radar-backend-kafka/radar.yml diff --git a/images/kafka-manager/README.md b/images/kafka-manager/README.md index cea3f60cf..bdf497fab 100644 --- a/images/kafka-manager/README.md +++ b/images/kafka-manager/README.md @@ -35,7 +35,6 @@ Instructions for deploying kafka-manager in a docker container and proxied throu ``` 12. Now start the stack with `dcompose-stack/radar-cp-hadoop-stack//install-radar-stack.sh`. This will build a docker image for kafka and start it in a container. You can access it with a browser at `https://host/kafkamanager/`. Open the link and add all the information. In this case the zookeeper host is at `zookeeper-1:2181`. This will look something like the image - -![Add a Cluster](/img/add_cluster.png) +![Add a Cluster](/images/kafka-manager/img/add_cluster.png) Note- You can also take the easy route and just pull the docker image from docker hub located at `radarcns/kafka-manager`. But remember that the context path is `/kafka-manager` so you will need to specify this in your `nginx.conf` file - diff --git a/images/radar-backend-kafka/Dockerfile b/images/radar-backend-kafka/Dockerfile deleted file mode 100644 index 8a101b64e..000000000 --- a/images/radar-backend-kafka/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -FROM confluentinc/cp-base:3.2.1 - -MAINTAINER Nivethika M , Joris B , Francesco N - -LABEL description="RADAR-base Backend streams and monitor" - -# Install RADAR-Backend -RUN echo "==> Installing Components" \ - # Download Git RADAR-Backend release - && echo "==> Downloading RADAR-base/RADAR-Backend 0.1.2 release from GitHub" \ - && cd /usr/local && mkdir RADAR-Backend - -ADD https://github.com/RADAR-base/RADAR-Backend/releases/download/0.1.2/radar-backend-0.1.2-all.jar /usr/share/java/ - -# Load topics validator -COPY ["./init.sh", "./kafka_status.sh", "/home/"] - -VOLUME /etc/radar.yml - -ENTRYPOINT ["./home/init.sh"] diff --git a/images/radar-backend-kafka/README.md b/images/radar-backend-kafka/README.md deleted file mode 100644 index 851573df5..000000000 --- a/images/radar-backend-kafka/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# Dockerised RADAR-Backend-Kafka - -It runs the RADAR-base Backend Kafka solution based on Kafka Streams 3.1.2, for more details about Kafka Streams click [here](http://docs.confluent.io/3.1.2/streams/index.html). - -Create the docker image: -``` -$ docker build -t radarcns/radar-backend-kafka ./ -``` - -Or pull from dockerhub: -``` -$ docker pull radarcns/radar-backend-kafka:0.1 -``` - -## Configuration - -Edit the radar.yml file to configure either the streams or the monitor. - -## Runtime environment variables - -This container requires two environment variable: - -- `KAFKA_REST_PROXY`: a valid Rest-Proxy instance -- `TOPIC_LIST`: a comma separated list containing all required topic names - -Before starting the streams, it waits until all topics inside TOPIC_LIST are available. This check is performed using the /topic Rest-Proxy API, for more details click here. - -## How to run - -For a complete use case scenario, check the RADAR-base `docker-compose` file available [here](https://github.com/RADAR-base/RADAR-Docker/blob/backend-integration/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml) diff --git a/images/radar-backend-kafka/init.sh b/images/radar-backend-kafka/init.sh deleted file mode 100755 index faaccf9f1..000000000 --- a/images/radar-backend-kafka/init.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# Busy waiting loop that waits untill all topic are available -echo "===> Waiting RADAR-base topics ... " -./home/kafka_status.sh - -# Start streams -echo "===> Starting " $1 "...." -./usr/bin/java -jar /usr/share/java/radar-backend-*.jar -c /etc/radar.yml $1 \ No newline at end of file diff --git a/images/radar-backend-kafka/kafka_status.sh b/images/radar-backend-kafka/kafka_status.sh deleted file mode 100755 index d00d1867b..000000000 --- a/images/radar-backend-kafka/kafka_status.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# Check if variables exist -if [ -z "$KAFKA_REST_PROXY" ]; then - echo "KAFKA_REST_PROXY is not defined" - exit 2 -fi - -if [ -z "$KAFKA_SCHEMA_REGISTRY" ]; then - echo "KAFKA_SCHEMA_REGISTRY is not defined" - exit 4 -fi - -KAFKA_BROKERS=${KAFKA_BROKERS:-3} - -max_timeout=32 - -tries=10 -timeout=1 -while true; do - IFS=, read -r -a array <<< $(curl -s "${KAFKA_REST_PROXY}/brokers" | sed 's/^.*\[\(.*\)\]\}/\1/') - LENGTH=${#array[@]} - if [ "$LENGTH" -eq "$KAFKA_BROKERS" ]; then - echo "Kafka brokers available." - break - fi - tries=$((tries - 1)) - if [ $tries -eq 0 ]; then - echo "FAILED: KAFKA BROKERs NOT READY." - exit 5 - fi - echo "Kafka brokers or Kafka REST proxy not ready. Retrying in ${timeout} seconds." - sleep $timeout - if [ $timeout -lt $max_timeout ]; then - timeout=$((timeout * 2)) - fi -done - -tries=10 -timeout=1 -while true; do - if wget --spider -q "${KAFKA_SCHEMA_REGISTRY}/subjects" 2>/dev/null; then - break - fi - tries=$((tries - 1)) - if [ $tries -eq 0 ]; then - echo "FAILED TO REACH SCHEMA REGISTRY. SCHEMAS NOT REGISTERED." - exit 6 - fi - echo "Failed to reach schema registry. Retrying in ${timeout} seconds." - sleep $timeout - if [ $timeout -lt $max_timeout ]; then - timeout=$((timeout * 2)) - fi -done - - -echo "Kafka is available. Ready to go!" diff --git a/images/radar-backend-kafka/radar.yml b/images/radar-backend-kafka/radar.yml deleted file mode 100644 index 18941a5a1..000000000 --- a/images/radar-backend-kafka/radar.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: 1.0 -released: 2016-11-27 - -#============================= Application =============================# -log_path: -#Possible value are standalone or high_performance -mode: standalone - -#============================== Zookeeper ==============================# -#List of Zookeeper instances -zookeeper: - - host: zookeeper-1 - port: 2181 - -#================================ Kafka ================================# -#List of Kafka brokers -broker: - - host: kafka-1 - port: 9092 - -#Kafka internal parameters -auto_commit_interval_ms: 1000 -session_timeout_ms: 10000 - -#============================ Kafka Streams ============================# -#The number of threads that a stream must be run according is priority -stream_priority: - low: 1 - normal: 2 - high: 4 - -#=========================== Schema Registry ===========================# -#List of Schema Registry instances -schema_registry: - - host: schema-registry-1 - port: 8081 - protocol: http \ No newline at end of file From 29b4d65dd30efa08f64c899b7a850b70f98b1537 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Fri, 23 Feb 2018 13:10:27 +0000 Subject: [PATCH 14/37] directory structure for sink connectors properties --- .travis.yml | 4 ++-- .../radar-cp-hadoop-stack/docker-compose.yml | 4 ++-- .../{ => hdfs-connector}/sink-hdfs-high.properties | 0 .../{ => hdfs-connector}/sink-hdfs-low.properties | 0 .../{ => hdfs-connector}/sink-hdfs-med.properties | 0 .../sink-hdfs.properties.template | 0 .../sink-mongo.properties.template | 0 .../radar-cp-hadoop-stack/install-radar-stack.sh | 14 +++++++------- 8 files changed, 11 insertions(+), 11 deletions(-) rename dcompose-stack/radar-cp-hadoop-stack/etc/{ => hdfs-connector}/sink-hdfs-high.properties (100%) rename dcompose-stack/radar-cp-hadoop-stack/etc/{ => hdfs-connector}/sink-hdfs-low.properties (100%) rename dcompose-stack/radar-cp-hadoop-stack/etc/{ => hdfs-connector}/sink-hdfs-med.properties (100%) rename dcompose-stack/radar-cp-hadoop-stack/etc/{ => hdfs-connector}/sink-hdfs.properties.template (100%) rename dcompose-stack/radar-cp-hadoop-stack/etc/{ => mongodb-connector}/sink-mongo.properties.template (100%) diff --git a/.travis.yml b/.travis.yml index 86699fc7b..1b61a01b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,8 +37,8 @@ script: - sudo docker volume create certs-data - cp etc/radar-backend/radar.yml.template etc/radar-backend/radar.yml - cp etc/webserver/nginx.conf.template etc/webserver/nginx.conf - - cp etc/sink-hdfs.properties.template etc/sink-hdfs.properties - - cp etc/sink-mongo.properties.template etc/sink-mongo.properties + - cp etc/hdfs-connector/sink-hdfs.properties.template etc/hdfs-connector/sink-hdfs.properties + - cp etc/mongodb-connector/sink-mongo.properties.template etc/mongodb-connector/sink-mongo.properties - sudo $HOME/bin/docker-compose up -d --build && sleep 15 && [ -z "$($HOME/bin/docker-compose ps | tail -n +3 | grep " Exit ")" ] - sudo $HOME/bin/docker-compose down - sudo docker network rm hadoop diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 0c84f7b17..4ccef40e3 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -359,7 +359,7 @@ services: image: radarbase/radar-mongodb-connector-auto:0.1 restart: on-failure volumes: - - ./etc/sink-mongo.properties:/etc/kafka-connect/sink.properties + - ./etc/mongodb-connector/sink-mongo.properties:/etc/kafka-connect/sink.properties networks: - zookeeper - kafka @@ -410,7 +410,7 @@ services: image: radarbase/radar-hdfs-connector-auto:0.1 restart: on-failure volumes: - - ./etc/sink-hdfs.properties:/etc/kafka-connect/sink-hdfs.properties + - ./etc/hdfs-connector/sink-hdfs.properties:/etc/kafka-connect/sink-hdfs.properties networks: - zookeeper - kafka diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-high.properties b/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-high.properties similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-high.properties rename to dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-high.properties diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-low.properties b/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-low.properties similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-low.properties rename to dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-low.properties diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-med.properties b/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-med.properties similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-med.properties rename to dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-med.properties diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs.properties.template b/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs.properties.template similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs.properties.template rename to dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs.properties.template diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/sink-mongo.properties.template b/dcompose-stack/radar-cp-hadoop-stack/etc/mongodb-connector/sink-mongo.properties.template similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/etc/sink-mongo.properties.template rename to dcompose-stack/radar-cp-hadoop-stack/etc/mongodb-connector/sink-mongo.properties.template diff --git a/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh b/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh index 10d6f7f2b..2e8b3f93c 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh @@ -16,8 +16,8 @@ check_config_present etc/radar-backend/radar.yml check_config_present etc/managementportal/config/oauth_client_details.csv check_config_present etc/rest-api/mp_info.yml check_config_present etc/redcap-integration/radar.yml -copy_template_if_absent etc/sink-mongo.properties -copy_template_if_absent etc/sink-hdfs.properties +copy_template_if_absent etc/mongodb-connector/sink-mongo.properties +copy_template_if_absent etc/hdfs-connector/sink-hdfs.properties copy_template_if_absent etc/rest-api/radar.yml copy_template_if_absent etc/rest-api/device-catalog.yml copy_template_if_absent etc/webserver/nginx.conf @@ -59,9 +59,9 @@ sudo-linux docker-compose run --rm kafka-init echo "==> Configuring MongoDB Connector" # Update sink-mongo.properties -inline_variable 'mongo.username=' $HOTSTORAGE_USERNAME etc/sink-mongo.properties -inline_variable 'mongo.password=' $HOTSTORAGE_PASSWORD etc/sink-mongo.properties -inline_variable 'mongo.database=' $HOTSTORAGE_NAME etc/sink-mongo.properties +inline_variable 'mongo.username=' $HOTSTORAGE_USERNAME etc/mongodb-connector/sink-mongo.properties +inline_variable 'mongo.password=' $HOTSTORAGE_PASSWORD etc/mongodb-connector/sink-mongo.properties +inline_variable 'mongo.database=' $HOTSTORAGE_NAME etc/mongodb-connector/sink-mongo.properties # Set topics if [ -z "${COMBINED_AGG_TOPIC_LIST}"]; then @@ -70,7 +70,7 @@ if [ -z "${COMBINED_AGG_TOPIC_LIST}"]; then COMBINED_AGG_TOPIC_LIST="${RADAR_AGG_TOPIC_LIST},${COMBINED_AGG_TOPIC_LIST}" fi fi -inline_variable 'topics=' "${COMBINED_AGG_TOPIC_LIST}" etc/sink-mongo.properties +inline_variable 'topics=' "${COMBINED_AGG_TOPIC_LIST}" etc/mongodb-connector/sink-mongo.properties echo "==> Configuring HDFS Connector" if [ -z "${COMBINED_RAW_TOPIC_LIST}"]; then @@ -79,7 +79,7 @@ if [ -z "${COMBINED_RAW_TOPIC_LIST}"]; then COMBINED_RAW_TOPIC_LIST="${RADAR_RAW_TOPIC_LIST},${COMBINED_RAW_TOPIC_LIST}" fi fi -inline_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/sink-hdfs.properties +inline_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/hdfs-connector/sink-hdfs.properties echo "==> Configuring Management Portal" From c223d992765f643e43f0e99a3b1ccf59ce36427b Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Fri, 23 Feb 2018 13:30:00 +0000 Subject: [PATCH 15/37] Use latest kafka mongodb connector --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 4ccef40e3..bcad8632f 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -356,7 +356,7 @@ services: # RADAR mongo connector # #---------------------------------------------------------------------------# radar-mongodb-connector: - image: radarbase/radar-mongodb-connector-auto:0.1 + image: radarbase/kafka-connect-mongodb-sink:0.2.0 restart: on-failure volumes: - ./etc/mongodb-connector/sink-mongo.properties:/etc/kafka-connect/sink.properties From 70f2d9ac8c62f78f674eea470a2bddabbc7f1f61 Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Fri, 23 Feb 2018 15:08:30 +0000 Subject: [PATCH 16/37] Use latest radar schema version --- dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile index 9f2bfd33e..924366ebc 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile +++ b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile @@ -25,7 +25,7 @@ RUN curl -#o /usr/share/java/avro-tools.jar \ "$(curl -s http://www.apache.org/dyn/closer.cgi/avro/\?as_json \ | jq --raw-output ".preferred")avro/avro-1.8.2/java/avro-tools-1.8.2.jar" -ENV RADAR_SCHEMAS_VERSION=0.2.3 +ENV RADAR_SCHEMAS_VERSION=0.3.1 RUN curl -#L https://github.com/RADAR-base/RADAR-Schemas/releases/download/v${RADAR_SCHEMAS_VERSION}/radar-schemas-tools-${RADAR_SCHEMAS_VERSION}.tar.gz \ | tar xz -C /usr --strip-components 1 From 90130be3cbb500af229f1949db9879ffa46cecf5 Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Fri, 23 Feb 2018 15:44:16 +0000 Subject: [PATCH 17/37] Add source statistic monitor config --- .../etc/radar-backend/radar.yml.template | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template index 459b1cfae..e3a5c4d2c 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template @@ -73,6 +73,27 @@ disconnect_monitor: - android_empatica_e4_temperature timeout: 1800 # seconds after which a stream is set disconnected alert_repetitions: 2 # number of additional emails to send after the first + +#====================== Source statistics monitor ======================# +statistics_monitors: + - name: Empatica E4 + topics: + - android_empatica_e4_blood_volume_pulse_1min + output_topic: source_statistics_empatica_e4 + - name: Biovotion VSM1 + topics: + - android_biovotion_vsm1_acceleration_1min + output_topic: source_statistics_biovotion_vsm1 + - name: RADAR pRMT + topics: + - android_phone_acceleration_1min + - android_phone_bluetooth_devices + - android_phone_sms + - android_phone_call + - android_phone_contacts + - android_phone_usage_event + - android_phone_relative_location + output_topic: source_statistics_radar_prmt #========================= Kafka stream ================================# stream_masters: From caf2cb3c5afbca8914e06ff5005f607bc230528f Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Sat, 24 Feb 2018 11:45:37 +0000 Subject: [PATCH 18/37] Fix mongodb connector properties --- .../etc/mongodb-connector/sink-mongo.properties.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/mongodb-connector/sink-mongo.properties.template b/dcompose-stack/radar-cp-hadoop-stack/etc/mongodb-connector/sink-mongo.properties.template index 428a101ff..071cdd178 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/mongodb-connector/sink-mongo.properties.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/mongodb-connector/sink-mongo.properties.template @@ -2,7 +2,7 @@ name=radar-connector-mongodb-sink # Kafka connector configuration -connector.class=org.radarcns.mongodb.MongoDbSinkConnector +connector.class=org.radarcns.connect.mongodb.MongoDbSinkConnector tasks.max=1 # Topics that will be consumed @@ -22,4 +22,4 @@ mongo.database= #mongo.collection.format={$topic} # Factory class to do the actual record conversion -record.converter.class=org.radarcns.sink.mongodb.RecordConverterFactoryRadar +record.converter.class=org.radarcns.connect.mongodb.serialization.RecordConverterFactory From b2a58d9aa619923a7dd2027e5540fadf0eda955e Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Sat, 24 Feb 2018 11:51:20 +0000 Subject: [PATCH 19/37] fix healthcheck --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index bcad8632f..1233ddac2 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -241,7 +241,7 @@ services: volumes: - "./etc/rest-api:/usr/local/tomcat/conf/radar" healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8080/radar/api/source/specification/ANDROID"] + test: ["CMD", "curl", "-f", "http://localhost:8080/radar/api/swagger.json"] interval: 1m timeout: 5s retries: 3 @@ -280,7 +280,7 @@ services: API_URI: https://${SERVER_NAME}/api BASE_HREF: /dashboard/ healthcheck: - test: ["CMD", "curl", "-IX", "HEAD", "http://localhost:80/"] + test: ["CMD", "wget", "-s", "http://localhost:80/"] interval: 1m timeout: 5s retries: 3 From 0f3254abd6216de37ba81ec441aed2c92af22a28 Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Mon, 26 Feb 2018 11:20:49 +0000 Subject: [PATCH 20/37] use --force for registering schemas --- .../radar-cp-hadoop-stack/kafka-radarinit/topic_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/topic_init.sh b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/topic_init.sh index 1c85cc8f0..e5d936a1b 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/topic_init.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/topic_init.sh @@ -32,7 +32,7 @@ while true; do fi done -if ! radar-schemas-tools register "${KAFKA_SCHEMA_REGISTRY}" merged; then +if ! radar-schemas-tools register --force "${KAFKA_SCHEMA_REGISTRY}" merged; then echo "FAILED TO REGISTER SCHEMAS" exit 1 fi From b2fd76b8d21aabcbaa6ef85de06c781d9518b170 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 27 Feb 2018 14:44:40 +0100 Subject: [PATCH 21/37] Fix docker initialization --- .../radar-cp-hadoop-stack/docker-compose.yml | 4 ++-- .../radar-cp-hadoop-stack/kafka-radarinit/init.sh | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index bcad8632f..21c1eed76 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -393,9 +393,9 @@ services: CONNECT_CONSUMER_MAX_POLL_INTERVAL_MS: 300000 CONNECT_CONSUMER_SESSION_TIMEOUT_MS: 10000 CONNECT_CONSUMER_HEARTBEAT_INTERVAL_MS: 3000 + CONNECT_PLUGIN_PATH: /usr/share/java/kafka-connect/plugins KAFKA_BROKERS: 3 - # For backwards compatibility - TOPIC_LIST: "application_record_counts" + CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" healthcheck: test: ["CMD-SHELL", "curl -sf localhost:8083/connectors/radar-connector-mongodb-sink/status | grep -o '\"state\":\"[^\"]*\"' | tr '\\n' ',' | grep -vq FAILED || exit 1"] interval: 1m diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh index c5e0b5213..5f62c56eb 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh @@ -7,10 +7,15 @@ rsync -a /schema/conf/ /schema/merged # Compiling updated schemas echo "Compiling schemas..." >&2 + +# Regex for schemas with a dependency that is a class +# e.g., a literal class starting with a capital, or +# a namespace with internal periods. +DEPENDENT_REGEX='"(items|type)": (\[\s*"null",\s*)?"([A-Z]|[^".]*\.)' # Separate enums so that they can be referenced in later files -read -r -a enums <<< $(find merged/commons -name "*.avsc" -exec grep -q '^ "type": "enum"' "{}" \; -print | tr '\n' ' ') -read -r -a notenums <<< $(find merged/commons -name "*.avsc" -exec grep -qv '^ "type": "enum"' "{}" \; -print | tr '\n' ' ') -java -jar /usr/share/java/avro-tools.jar compile -string schema ${enums[@]} ${notenums[@]} java/src 2>/dev/null +read -r -a notdependent <<< $(find merged/commons -name "*.avsc" -exec grep -Eqv "$DEPENDENT_REGEX" "{}" \; -print | tr '\n' ' ') +read -r -a dependent <<< $(find merged/commons -name "*.avsc" -exec grep -Eq "$DEPENDENT_REGEX" \; -print | tr '\n' ' ') +java -jar /usr/share/java/avro-tools.jar compile -string schema ${notdependent[@]} ${dependent[@]} java/src 2>/dev/null find java/src -name "*.java" -print0 | xargs -0 javac -cp /usr/lib/*:java/classes -d java/classes -sourcepath java/src # Update the radar schemas so the tools find the new classes in classpath jar uf /usr/lib/radar-schemas-commons-${RADAR_SCHEMAS_VERSION}.jar -C java/classes . From e6c46f4ba7e04b42efdb26f526b8546e59ab4cc3 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 28 Feb 2018 14:34:55 +0100 Subject: [PATCH 22/37] Fix kafka init grep arguments --- dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh index 5f62c56eb..4e7c5d5be 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh @@ -14,7 +14,7 @@ echo "Compiling schemas..." >&2 DEPENDENT_REGEX='"(items|type)": (\[\s*"null",\s*)?"([A-Z]|[^".]*\.)' # Separate enums so that they can be referenced in later files read -r -a notdependent <<< $(find merged/commons -name "*.avsc" -exec grep -Eqv "$DEPENDENT_REGEX" "{}" \; -print | tr '\n' ' ') -read -r -a dependent <<< $(find merged/commons -name "*.avsc" -exec grep -Eq "$DEPENDENT_REGEX" \; -print | tr '\n' ' ') +read -r -a dependent <<< $(find merged/commons -name "*.avsc" -exec grep -Eq "$DEPENDENT_REGEX" "{}" \; -print | tr '\n' ' ') java -jar /usr/share/java/avro-tools.jar compile -string schema ${notdependent[@]} ${dependent[@]} java/src 2>/dev/null find java/src -name "*.java" -print0 | xargs -0 javac -cp /usr/lib/*:java/classes -d java/classes -sourcepath java/src # Update the radar schemas so the tools find the new classes in classpath From 85dbac7a61d0bb426feab85c41f12616a082c936 Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Thu, 1 Mar 2018 15:42:14 +0000 Subject: [PATCH 23/37] Use latest schema version --- dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile index 924366ebc..4a84d7f31 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile +++ b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile @@ -25,7 +25,7 @@ RUN curl -#o /usr/share/java/avro-tools.jar \ "$(curl -s http://www.apache.org/dyn/closer.cgi/avro/\?as_json \ | jq --raw-output ".preferred")avro/avro-1.8.2/java/avro-tools-1.8.2.jar" -ENV RADAR_SCHEMAS_VERSION=0.3.1 +ENV RADAR_SCHEMAS_VERSION=0.3.2 RUN curl -#L https://github.com/RADAR-base/RADAR-Schemas/releases/download/v${RADAR_SCHEMAS_VERSION}/radar-schemas-tools-${RADAR_SCHEMAS_VERSION}.tar.gz \ | tar xz -C /usr --strip-components 1 From 6ef8ad7564b0ee89dd91a670c88c41118a3afa5c Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Thu, 1 Mar 2018 15:43:11 +0000 Subject: [PATCH 24/37] Change topic name acc to latest schema --- .../radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template index e3a5c4d2c..cdd72ce65 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template @@ -93,7 +93,7 @@ statistics_monitors: - android_phone_contacts - android_phone_usage_event - android_phone_relative_location - output_topic: source_statistics_radar_prmt + output_topic: source_statistics_android_phone #========================= Kafka stream ================================# stream_masters: From 31a3283c37f522bf0d91eac526836c3b7dca3b03 Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Thu, 1 Mar 2018 17:44:28 +0000 Subject: [PATCH 25/37] Update oauth_client_details.csv.template --- .../managementportal/config/oauth_client_details.csv.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/config/oauth_client_details.csv.template b/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/config/oauth_client_details.csv.template index b5f4b8587..ad005bea3 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/config/oauth_client_details.csv.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/config/oauth_client_details.csv.template @@ -2,6 +2,6 @@ client_id;resource_ids;client_secret;scope;authorized_grant_types;redirect_uri;a pRMT;res_ManagementPortal,res_gateway;secret;MEASUREMENT.CREATE,SUBJECT.UPDATE,SUBJECT.READ,PROJECT.READ,SOURCETYPE.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ,USER.READ,ROLE.READ;refresh_token,authorization_code;;;43200;7948800;{"dynamic_registration": true}; aRMT;res_ManagementPortal,res_gateway;secret;MEASUREMENT.CREATE,SUBJECT.UPDATE,SUBJECT.READ,PROJECT.READ,SOURCETYPE.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ,USER.READ,ROLE.READ;refresh_token,authorization_code;;;43200;7948800;{"dynamic_registration": true}; THINC-IT;res_ManagementPortal,res_gateway;secret;MEASUREMENT.CREATE,SUBJECT.UPDATE,SUBJECT.READ,PROJECT.READ,SOURCETYPE.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ,USER.READ,ROLE.READ;refresh_token,authorization_code;;;43200;7948800;{"dynamic_registration": true}; -radar_restapi;res_ManagementPortal;secret;SUBJECT.READ,PROJECT.READ,SOURCE.READ,SOURCETYPE.READ;client_credentials;;;43200;259200;{}; +radar_restapi;res_ManagementPortal;secret;SUBJECT.READ,PROJECT.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ;client_credentials;;;43200;259200;{}; radar_redcap_integrator;res_ManagementPortal;secret;PROJECT.READ,SUBJECT.CREATE,SUBJECT.READ,SUBJECT.UPDATE;client_credentials;;;43200;259200;{}; -radar_dashboard;res_ManagementPortal,res_restApi;secret;SUBJECT.READ,PROJECT.READ,SOURCE.READ,SOURCETYPE.READ;client_credentials;;;43200;259200;{}; +radar_dashboard;res_ManagementPortal,res_RestApi;secret;SUBJECT.READ,PROJECT.READ,SOURCE.READ,SOURCETYPE.READ,MEASUREMENT.READ;client_credentials;;;43200;259200;{}; From 6bd1cffc4eae2971981ce17e9127d1b3840d827c Mon Sep 17 00:00:00 2001 From: nivethika Date: Fri, 2 Mar 2018 18:50:41 +0100 Subject: [PATCH 26/37] update rest-api 0.2.0 --- .../radar-cp-hadoop-stack/docker-compose.yml | 8 +- .../etc/rest-api/device-catalog.yml.template | 116 ------------------ .../etc/rest-api/mp_info.yml.template | 10 -- .../etc/rest-api/radar.yml.template | 47 +++---- .../etc/webserver/nginx.conf.template | 2 +- .../install-radar-stack.sh | 11 +- 6 files changed, 34 insertions(+), 160 deletions(-) delete mode 100644 dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/device-catalog.yml.template delete mode 100644 dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/mp_info.yml.template diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index fd6c9fb67..a03dc14d2 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -231,17 +231,19 @@ services: # RADAR REST API # #---------------------------------------------------------------------------# rest-api: - image: radarbase/radar-restapi:0.1.1 + image: radarbase/radar-restapi:0.2.0 networks: - hotstorage - api + - management depends_on: - hotstorage + - managementportal-app restart: always volumes: - - "./etc/rest-api:/usr/local/tomcat/conf/radar" + - "./etc/rest-api:/usr/local/conf/radar/rest-api" healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8080/radar/api/swagger.json"] + test: ["CMD", "curl", "-f", "http://localhost:8080/api/openapi.json"] interval: 1m timeout: 5s retries: 3 diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/device-catalog.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/device-catalog.yml.template deleted file mode 100644 index 00ff3031b..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/device-catalog.yml.template +++ /dev/null @@ -1,116 +0,0 @@ -#=========================== Device Catalog ===========================# -#List of integrated sources with relative configuration -device_catalog: - #====================== Empatica E4 Wristband =======================# - #A device item states the specification and the configurations required to integrate a wearable - # device into the data access interface. The used value must be contained in - # RADAR-Schemas/restapi/source/sources_type.avsc - EMPATICA: - #List of on-board sensors - sensors: - #Sensor name. Possible values are listed by RADAR-Schemas/restapi/data/sensor_type.avsc - - name: ACCELEROMETER - #Default data frequency - frequency: 32.0 - #Measurement unit. Possible values are listed in RADAR-Schemas/restapi/data/unit.avsc - unit: G - #Define if the samples are RAW data or instead they the result of some computation. Possible - # values are listed by RADAR-Schemas/restapi/data/data_type.avsc - data_type: RAW - #The associated Data Class: actual generalised reader implantation that extract data from - # the storage - data_class: ACCELERATION_FORMAT - #List of supported collections - collections: - #Time-frame of 10 seconds - 10sec: android_empatica_e4_acceleration_output - #Time-frame of 30 seconds - 30sec: android_empatica_e4_acceleration_output_30sec - #Time-frame of 1 minute - 1min: android_empatica_e4_acceleration_output_1min - #Time-frame of 10 minutes - 10min: android_empatica_e4_acceleration_output_10min - #Time-frame of 1 hour - 1h: android_empatica_e4_acceleration_output_1h - #Time-frame of 1 day - 1d: android_empatica_e4_acceleration_output_1d - #Time-frame of 1 week - 1w: android_empatica_e4_acceleration_output_1w - - name: BATTERY - frequency: 1.0 - unit: PERCENTAGE - data_type: RAW - data_class: DOUBLE_FORMAT - collections: - 10sec: android_empatica_e4_battery_level_output - 30sec: android_empatica_e4_battery_level_output_30sec - 1min: android_empatica_e4_battery_level_output_1min - 10min: android_empatica_e4_battery_level_output_10min - 1h: android_empatica_e4_battery_level_output_1h - 1d: android_empatica_e4_battery_level_output_1d - 1w: android_empatica_e4_battery_level_output_1w - - name: BLOOD_VOLUME_PULSE - frequency: 64.0 - unit: NANOWATT - data_type: RAW - data_class: DOUBLE_FORMAT - collections: - 10sec: android_empatica_e4_blood_volume_pulse_output - 30sec: android_empatica_e4_blood_volume_pulse_output_30sec - 1min: android_empatica_e4_blood_volume_pulse_output_1min - 10min: android_empatica_e4_blood_volume_pulse_output_10min - 1h: android_empatica_e4_blood_volume_pulse_output_1h - 1d: android_empatica_e4_blood_volume_pulse_output_1d - 1w: android_empatica_e4_blood_volume_pulse_output_1w - - name: ELECTRODERMAL_ACTIVITY - frequency: 4.0 - unit: MICROSIEMENS - data_type: RAW - data_class: DOUBLE_FORMAT - collections: - 10sec: android_empatica_e4_electrodermal_activity_output - 30sec: android_empatica_e4_electrodermal_activity_output_30sec - 1min: android_empatica_e4_electrodermal_activity_output_1min - 10min: android_empatica_e4_electrodermal_activity_output_10min - 1h: android_empatica_e4_electrodermal_activity_output_1h - 1d: android_empatica_e4_electrodermal_activity_output_1d - 1w: android_empatica_e4_electrodermal_activity_output_1w - - name: HEART_RATE - frequency: 1.0 - unit: BEATS_PER_MIN - data_type: RADAR - data_class: DOUBLE_FORMAT - collections: - 10sec: android_empatica_e4_heartrate - 30sec: android_empatica_e4_heartrate_30sec - 1min: android_empatica_e4_heartrate_1min - 10min: android_empatica_e4_heartrate_10min - 1h: android_empatica_e4_heartrate_1h - 1d: android_empatica_e4_heartrate_1d - 1w: android_empatica_e4_heartrate_1w - - name: INTER_BEAT_INTERVAL - frequency: 1.0 - unit: BEATS_PER_MIN - data_type: RAW - data_class: DOUBLE_FORMAT - collections: - 10sec: android_empatica_e4_inter_beat_interval_output - 30sec: android_empatica_e4_inter_beat_interval_output_30sec - 1min: android_empatica_e4_inter_beat_interval_output_1min - 10min: android_empatica_e4_inter_beat_interval_output_10min - 1h: android_empatica_e4_inter_beat_interval_output_1h - 1d: android_empatica_e4_inter_beat_interval_output_1d - 1w: android_empatica_e4_inter_beat_interval_output_1w - - name: THERMOMETER - frequency: 4.0 - unit: CELSIUS - data_type: RAW - data_class: DOUBLE_FORMAT - collections: - 10sec: android_empatica_e4_temperature_output - 30sec: android_empatica_e4_temperature_output_30sec - 1min: android_empatica_e4_temperature_output_1min - 10min: android_empatica_e4_temperature_output_10min - 1h: android_empatica_e4_temperature_output_1h - 1d: android_empatica_e4_temperature_output_1d - 1w: android_empatica_e4_temperature_output_1w \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/mp_info.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/mp_info.yml.template deleted file mode 100644 index ed71a0e01..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/mp_info.yml.template +++ /dev/null @@ -1,10 +0,0 @@ -version: 0.1-alpha -released: 2017-08-29 - -# ManagementPortal configuration -oauth_client_id: #OAuth2 clientId used by the webApp for making requests -oauth_client_secret: #OAuth2 client secret -management_portal_url: #URL pointing Management Portal -token_endpoint: #URL managing tokens -project_endpoint: #URL managing project function -subject_endpoint: #URL managing subject functions diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/radar.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/radar.yml.template index 70aa68423..3a97db6e5 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/radar.yml.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/radar.yml.template @@ -1,27 +1,30 @@ -version: 0.1 -released: 2017-05-10 -# Swagger documentation version -swagger_version: 1.0.0 - -# Comma separated list stating supported Application Protocols applicable to query APIs -application_protocol: https - -# Machine address hosting the APIs -host: localhost - -# Base or Root path for the RESTFul interface -api_base_path: api - -# Mongodb hosts -mongo_hosts: +# Mongodb hosts a map of hostname and port +mongodb_hosts: hotstorage: 27017 # Mongodb User -mongo_user: - usr: - pwd: - db: +mongodb_credentials: + username: + password: + database_name: + +management_portal_config: + oauth_client_id: radar_restapi #OAuth2 clientId used by the webApp for making requests + oauth_client_secret: secret #OAuth2 client secret + oauth_client_scopes: SUBJECT.READ PROJECT.READ SOURCE.READ SOURCETYPE.READ SOURCEDATA.READ + management_portal_url: http://managementportal-app:8080/managementportal/ #URL pointing Management Portal + token_endpoint: oauth/token #URL managing tokens + project_endpoint: api/projects/ #URL managing project function + subject_endpoint: api/subjects/ + source_type_endpoint: api/source-types/ + source_data_endpoint: api/source-data/ + source_endpoint: api/sources/ -# Path to the Device Catalog -device_catalog: device-catalog.yml \ No newline at end of file +# Timeout duration for every source-type to decide source status whether its connected or not. +# A source-type should be defined by following the convention of producer_model as mentioned in +# the specification in radar-schemas +# timeout should be specified as the ISO-8601 duration format {@code PnDTnHnMn.nS}. +source-type-connection-timeout: + android_phone: PT2H + empatica_e4: PT1H \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template index 05e494711..cc9a28043 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template @@ -99,7 +99,7 @@ http { } location /api/ { include cors.conf; - proxy_pass http://rest-api:8080/radar/api/; + proxy_pass http://rest-api:8080/api/; proxy_set_header Host $host; } location /redcapint/ { diff --git a/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh b/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh index 2e8b3f93c..c477e0de0 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh @@ -14,12 +14,10 @@ fi check_config_present .env etc/env.template check_config_present etc/radar-backend/radar.yml check_config_present etc/managementportal/config/oauth_client_details.csv -check_config_present etc/rest-api/mp_info.yml check_config_present etc/redcap-integration/radar.yml copy_template_if_absent etc/mongodb-connector/sink-mongo.properties copy_template_if_absent etc/hdfs-connector/sink-hdfs.properties copy_template_if_absent etc/rest-api/radar.yml -copy_template_if_absent etc/rest-api/device-catalog.yml copy_template_if_absent etc/webserver/nginx.conf . ./.env @@ -100,12 +98,9 @@ fi echo "==> Configuring REST-API" # Set MongoDb credential -inline_variable 'usr:[[:space:]]' "$HOTSTORAGE_USERNAME" etc/rest-api/radar.yml -inline_variable 'pwd:[[:space:]]' "$HOTSTORAGE_PASSWORD" etc/rest-api/radar.yml -inline_variable 'db:[[:space:]]' "$HOTSTORAGE_NAME" etc/rest-api/radar.yml - -# Set variable for Swagger -inline_variable 'host:[[:space:]]*' "${SERVER_NAME}" etc/rest-api/radar.yml +inline_variable 'username:[[:space:]]' "$HOTSTORAGE_USERNAME" etc/rest-api/radar.yml +inline_variable 'password:[[:space:]]' "$HOTSTORAGE_PASSWORD" etc/rest-api/radar.yml +inline_variable 'database_name:[[:space:]]' "$HOTSTORAGE_NAME" etc/rest-api/radar.yml echo "==> Configuring REDCap-Integration" From 455660828a176ded788dd5860e439df191d1f1ca Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Mon, 5 Mar 2018 18:28:55 +0000 Subject: [PATCH 27/37] Fix Streams --- .../etc/radar-backend/radar.yml.template | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template index cdd72ce65..29bb78511 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template @@ -20,8 +20,9 @@ broker: #Kafka internal parameters stream_properties: auto_commit_interval_ms: 1000 - session_timeout_ms: 10000 - + max.request.size: 3500042 #Set message.max.bytes for kafka brokers higher than or equal to this value + retries: 15 + session_timeout_ms: 20000 #============================ Kafka Streams ============================# #The number of threads that a stream must be run according is priority stream_priority: From 4e44ae28a39f33eb85be1cee7a372a7ae2cae245 Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Mon, 5 Mar 2018 18:30:43 +0000 Subject: [PATCH 28/37] Increase kafka max message size --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index fd6c9fb67..dd545df60 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -80,6 +80,7 @@ services: KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_LOG_RETENTION_HOURS: 730 + KAFKA_MESSAGE_MAX_BYTES: 4000048 KAFKA_LOG4J_LOGGERS: kafka.producer.async.DefaultEventHandler=INFO,kafka.controller=INFO,state.change.logger=INFO healthcheck: test: ["CMD-SHELL", "echo dump | nc zookeeper-1 2181 | grep -q /brokers/ids/1 || exit 1"] @@ -103,6 +104,7 @@ services: KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:9092 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_LOG_RETENTION_HOURS: 730 + KAFKA_MESSAGE_MAX_BYTES: 4000048 KAFKA_LOG4J_LOGGERS: kafka.producer.async.DefaultEventHandler=INFO,kafka.controller=INFO,state.change.logger=INFO healthcheck: test: ["CMD-SHELL", "echo dump | nc zookeeper-1 2181 | grep -q /brokers/ids/2 || exit 1"] @@ -126,6 +128,7 @@ services: KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:9092 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_LOG_RETENTION_HOURS: 730 + KAFKA_MESSAGE_MAX_BYTES: 4000048 KAFKA_LOG4J_LOGGERS: kafka.producer.async.DefaultEventHandler=INFO,kafka.controller=INFO,state.change.logger=INFO healthcheck: test: ["CMD-SHELL", "echo dump | nc zookeeper-1 2181 | grep -q /brokers/ids/3 || exit 1"] From f40579e0835eec660d1f93b89fb5de24b4400d75 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 28 Mar 2018 12:51:12 +0200 Subject: [PATCH 29/37] Ensure that kafka init identifies dependent schemas (fixes #116) --- .../radar-cp-hadoop-stack/kafka-radarinit/init.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh index 4e7c5d5be..f1cffed12 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh @@ -12,10 +12,19 @@ echo "Compiling schemas..." >&2 # e.g., a literal class starting with a capital, or # a namespace with internal periods. DEPENDENT_REGEX='"(items|type)": (\[\s*"null",\s*)?"([A-Z]|[^".]*\.)' + +find merged/commons -name "*.avsc" -print | sort > merged/file_list +find merged/commons -name "*.avsc" -exec grep -Eq "$DEPENDENT_REGEX" "{}" \; -print | sort > merged/file_list_dependent +comm -23 merged/file_list merged/file_list_dependent > merged/file_list_independent + # Separate enums so that they can be referenced in later files -read -r -a notdependent <<< $(find merged/commons -name "*.avsc" -exec grep -Eqv "$DEPENDENT_REGEX" "{}" \; -print | tr '\n' ' ') -read -r -a dependent <<< $(find merged/commons -name "*.avsc" -exec grep -Eq "$DEPENDENT_REGEX" "{}" \; -print | tr '\n' ' ') -java -jar /usr/share/java/avro-tools.jar compile -string schema ${notdependent[@]} ${dependent[@]} java/src 2>/dev/null +read -r -a independent <<< $(tr '\n' ' ' < merged/file_list_independent) +read -r -a dependent <<< $(tr '\n' ' ' < merged/file_list_dependent) + +printf "===> Independent schemas:\n$(echo ${independent[@]})\n" +printf "===> Dependent schemas:\n$(echo ${dependent[@]})\n" + +java -jar /usr/share/java/avro-tools.jar compile -string schema ${independent[@]} ${dependent[@]} java/src 2>/dev/null find java/src -name "*.java" -print0 | xargs -0 javac -cp /usr/lib/*:java/classes -d java/classes -sourcepath java/src # Update the radar schemas so the tools find the new classes in classpath jar uf /usr/lib/radar-schemas-commons-${RADAR_SCHEMAS_VERSION}.jar -C java/classes . From f037ae053083af9cd12768010d5f78d2aae11563 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 28 Mar 2018 13:27:13 +0200 Subject: [PATCH 30/37] Some simplification --- .../kafka-radarinit/init.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh index f1cffed12..7e3a16042 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh @@ -2,6 +2,8 @@ set -e +AVRO_TOOLS=/usr/share/java/avro-tools.jar + rsync -a /schema/original/commons /schema/original/specifications /schema/merged rsync -a /schema/conf/ /schema/merged @@ -13,18 +15,16 @@ echo "Compiling schemas..." >&2 # a namespace with internal periods. DEPENDENT_REGEX='"(items|type)": (\[\s*"null",\s*)?"([A-Z]|[^".]*\.)' -find merged/commons -name "*.avsc" -print | sort > merged/file_list -find merged/commons -name "*.avsc" -exec grep -Eq "$DEPENDENT_REGEX" "{}" \; -print | sort > merged/file_list_dependent -comm -23 merged/file_list merged/file_list_dependent > merged/file_list_independent - # Separate enums so that they can be referenced in later files -read -r -a independent <<< $(tr '\n' ' ' < merged/file_list_independent) -read -r -a dependent <<< $(tr '\n' ' ' < merged/file_list_dependent) +find merged/commons -name "*.avsc" -print | sort > merged/file_list +DEPENDENT=$(find merged/commons -name "*.avsc" -exec grep -Eq "$DEPENDENT_REGEX" "{}" \; -print | sort) +# remove dependent files from all files to get independent files +INDEPENDENT=$(printf "${DEPENDENT}" | comm -23 merged/file_list -) -printf "===> Independent schemas:\n$(echo ${independent[@]})\n" -printf "===> Dependent schemas:\n$(echo ${dependent[@]})\n" +printf "===> Independent schemas:\n${INDEPENDENT}\n" +printf "===> Dependent schemas:\n${DEPENDENT}\n" -java -jar /usr/share/java/avro-tools.jar compile -string schema ${independent[@]} ${dependent[@]} java/src 2>/dev/null +java -jar "${AVRO_TOOLS}" compile -string schema ${INDEPENDENT} ${DEPENDENT} java/src 2>/dev/null find java/src -name "*.java" -print0 | xargs -0 javac -cp /usr/lib/*:java/classes -d java/classes -sourcepath java/src # Update the radar schemas so the tools find the new classes in classpath jar uf /usr/lib/radar-schemas-commons-${RADAR_SCHEMAS_VERSION}.jar -C java/classes . From 46aa72cebc77e86720949e031f7b817a46b33251 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 16 Apr 2018 09:36:25 +0200 Subject: [PATCH 31/37] Fixed duplicate cors header --- .../radar-cp-hadoop-stack/etc/webserver/cors.conf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/cors.conf b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/cors.conf index 6abf87141..abc910403 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/cors.conf +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/cors.conf @@ -15,6 +15,10 @@ # Forked from this Gist: https://gist.github.com/michiel/1064640 # +# do not send duplicate origin headers if the underlying +# service is CORS-compliant +proxy_hide_header 'Access-Control-Allow-Origin'; + set $cors_method ''; if ($request_method = 'GET') { @@ -48,6 +52,4 @@ if ($cors_method = 'noopt') { add_header 'Access-Control-Allow-Credentials' 'true' always; add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always; add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With' always; - # required to be able to read Authorization header in frontend - #add_header 'Access-Control-Expose-Headers' 'Authorization' always; } From e6ad3bbaccd8f8f2673c2843d869c273ef53928e Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Tue, 1 May 2018 23:40:02 +0100 Subject: [PATCH 32/37] Minor updates --- .../radar-cp-hadoop-stack/docker-compose.yml | 2 +- images/radar-restapi/Dockerfile | 27 ------------------- images/radar-restapi/README.md | 20 -------------- 3 files changed, 1 insertion(+), 48 deletions(-) delete mode 100644 images/radar-restapi/Dockerfile delete mode 100644 images/radar-restapi/README.md diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index a03dc14d2..0fc88f164 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -243,7 +243,7 @@ services: volumes: - "./etc/rest-api:/usr/local/conf/radar/rest-api" healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8080/api/openapi.json"] + test: ["CMD", "wget", "--spider", "http://localhost:8080/api/openapi.json"] interval: 1m timeout: 5s retries: 3 diff --git a/images/radar-restapi/Dockerfile b/images/radar-restapi/Dockerfile deleted file mode 100644 index 567dea665..000000000 --- a/images/radar-restapi/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -FROM tomcat:8.0.44-jre8 - -ENV JAVA_OPTS=-Djava.security.egd=file:/dev/urandom - -MAINTAINER @fnobilia, @afolarin - -LABEL description="RADAR-base Rest Api docker container" - -# Install Rest API -ADD https://github.com/RADAR-base/RADAR-RestApi/releases/download/v0.1.1/radar-0.1.1.war /usr/local/tomcat/webapps/radar.war - -EXPOSE 8080 - -CMD ["catalina.sh", "run"] diff --git a/images/radar-restapi/README.md b/images/radar-restapi/README.md deleted file mode 100644 index 0792e25c6..000000000 --- a/images/radar-restapi/README.md +++ /dev/null @@ -1,20 +0,0 @@ -## Dockerised RADAR-RestApi - -Create the docker image: -``` -$ docker build -t radarcns/radar-restapi ./ -``` - -Or pull from dockerhub: -``` -$ docker pull radarcns/radar-restapi:latest -``` - -Edit `radar.yml` and `device-catalog.yml`, and place them under `/path/to/config` - -Run the docker image locally: -``` -$ docker run -d -p 8080:8080 -v /path/to/config:/usr/local/tomcat/conf/radar --name radar-restapi radarcns/radar-restapi:0.1.1 -``` - -The RestApi will be running at http://localhost:8080. To test them, access the [Swagger Documentation](http://localhost:8080/radar/api/swagger.json) From affcc40450c9b0c96e3894257913a784814b78e8 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Tue, 8 May 2018 15:13:10 +0100 Subject: [PATCH 33/37] Move to confluent 4.1.0 and update other components to latest --- .../radar-cp-hadoop-stack/docker-compose.yml | 23 +++++++++++-------- .../radar-cp-hadoop-stack/docker-prune.sh | 22 +++++++++++------- .../radar-cp-hadoop-stack/etc/.gitignore | 7 +++--- .../sink-mongo.properties.template | 3 +++ .../radar-cp-hadoop-stack/hdfs_restructure.sh | 2 +- 5 files changed, 34 insertions(+), 23 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index b1eb2a5dd..0ae5d9514 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -41,7 +41,7 @@ services: # Zookeeper Cluster # #---------------------------------------------------------------------------# zookeeper-1: - image: confluentinc/cp-zookeeper:3.2.1-5 + image: confluentinc/cp-zookeeper:4.1.0 networks: - zookeeper volumes: @@ -65,7 +65,7 @@ services: # Kafka Cluster # #---------------------------------------------------------------------------# kafka-1: - image: confluentinc/cp-kafka:3.2.1-5 + image: confluentinc/cp-kafka:4.1.0 networks: - kafka - zookeeper @@ -89,7 +89,7 @@ services: retries: 3 kafka-2: - image: confluentinc/cp-kafka:3.2.1-5 + image: confluentinc/cp-kafka:4.1.0 networks: - kafka - zookeeper @@ -113,7 +113,7 @@ services: retries: 3 kafka-3: - image: confluentinc/cp-kafka:3.2.1-5 + image: confluentinc/cp-kafka:4.1.0 networks: - kafka - zookeeper @@ -140,7 +140,7 @@ services: # Schema Registry # #---------------------------------------------------------------------------# schema-registry-1: - image: confluentinc/cp-schema-registry:3.2.1-5 + image: confluentinc/cp-schema-registry:4.1.0 networks: - kafka - zookeeper @@ -164,7 +164,7 @@ services: # REST proxy # #---------------------------------------------------------------------------# rest-proxy-1: - image: confluentinc/cp-kafka-rest:3.2.1-5 + image: confluentinc/cp-kafka-rest:4.1.0 networks: - kafka - zookeeper @@ -361,7 +361,7 @@ services: # RADAR mongo connector # #---------------------------------------------------------------------------# radar-mongodb-connector: - image: radarbase/kafka-connect-mongodb-sink:0.2.0 + image: radarbase/kafka-connect-mongodb-sink:0.2.1 restart: on-failure volumes: - ./etc/mongodb-connector/sink-mongo.properties:/etc/kafka-connect/sink.properties @@ -400,6 +400,7 @@ services: CONNECT_CONSUMER_HEARTBEAT_INTERVAL_MS: 3000 CONNECT_PLUGIN_PATH: /usr/share/java/kafka-connect/plugins KAFKA_BROKERS: 3 + CONNECT_LOG4J_ROOT_LOGLEVEL: WARN CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" healthcheck: test: ["CMD-SHELL", "curl -sf localhost:8083/connectors/radar-connector-mongodb-sink/status | grep -o '\"state\":\"[^\"]*\"' | tr '\\n' ',' | grep -vq FAILED || exit 1"] @@ -450,6 +451,8 @@ services: CONNECTOR_PROPERTY_FILE_PREFIX: "sink-hdfs" KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" KAFKA_BROKERS: 3 + CONNECT_LOG4J_ROOT_LOGLEVEL: WARN + CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" # For backwards compatibility TOPIC_LIST: "application_record_counts" healthcheck: @@ -462,7 +465,7 @@ services: # RADAR backend streams # #---------------------------------------------------------------------------# radar-backend-stream: - image: radarbase/radar-backend:0.2.1 + image: radarbase/radar-backend:0.2.3 command: - stream networks: @@ -489,7 +492,7 @@ services: # RADAR backend monitor # #---------------------------------------------------------------------------# radar-backend-monitor: - image: radarbase/radar-backend:0.2.1 + image: radarbase/radar-backend:0.2.3 command: monitor networks: - zookeeper @@ -575,7 +578,7 @@ services: # Management Portal # #---------------------------------------------------------------------------# managementportal-app: - image: radarbase/management-portal:0.3.4 + image: radarbase/management-portal:0.3.6 networks: - default - api diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-prune.sh b/dcompose-stack/radar-cp-hadoop-stack/docker-prune.sh index 2d1a33172..5dc1c228f 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-prune.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-prune.sh @@ -5,11 +5,17 @@ check_command_exists docker -sudo-linux docker system prune --filter "label!=certs" "$@" && \ - sudo-linux rm -rf "$HDFS_DATA_DIR_1" && \ - sudo-linux rm -rf "$HDFS_DATA_DIR_2" && \ - sudo-linux rm -rf "$HDFS_NAME_DIR_1" && \ - sudo-linux rm -rf "$HDFS_NAME_DIR_2" && \ - sudo-linux rm -rf "$MONGODB_DIR" && \ - sudo-linux rm -rf "$MP_POSTGRES_DIR" - +echo "This will delete all the data associated with RADAR in HDFS, MongoDB and Postgres. This will also delete all data in docker containers. Do you wish to continue? yes or no." +select yn in "Yes" "No"; do + case $yn in + Yes ) sudo-linux docker system prune --filter "label!=certs" "$@"; + sudo-linux rm -rf "$HDFS_DATA_DIR_1"; + sudo-linux rm -rf "$HDFS_DATA_DIR_2"; + sudo-linux rm -rf "$HDFS_NAME_DIR_1"; + sudo-linux rm -rf "$HDFS_NAME_DIR_2"; + sudo-linux rm -rf "$MONGODB_DIR"; + sudo-linux rm -rf "$MP_POSTGRES_DIR"; + break;; + No ) exit;; + esac +done diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore index 394404aff..8d3a50299 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore @@ -1,7 +1,7 @@ -/sink-mongo.properties -/sink-hdfs.properties +/mongodb-connector/sink-mongo.properties +/hdfs-connector/sink-hdfs.properties /webserver/nginx.conf -/radar.yml +/radar-backend/radar.yml /smtp.env /rest-api/device-catalog.yml /rest-api/radar.yml @@ -11,4 +11,3 @@ /managementportal/config/liquibase/oauth_client_details.csv /schema/commons/ /schema/specifications/ - diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/mongodb-connector/sink-mongo.properties.template b/dcompose-stack/radar-cp-hadoop-stack/etc/mongodb-connector/sink-mongo.properties.template index 071cdd178..b5ad3cfe3 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/mongodb-connector/sink-mongo.properties.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/mongodb-connector/sink-mongo.properties.template @@ -21,5 +21,8 @@ mongo.database= # by the Kafka topic name. #mongo.collection.format={$topic} +# Buffer Capacity for mogodb writer. Default value is 20000 +# buffer.capacity=20000 + # Factory class to do the actual record conversion record.converter.class=org.radarcns.connect.mongodb.serialization.RecordConverterFactory diff --git a/dcompose-stack/radar-cp-hadoop-stack/hdfs_restructure.sh b/dcompose-stack/radar-cp-hadoop-stack/hdfs_restructure.sh index a7a3c6a7f..345468ddf 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/hdfs_restructure.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/hdfs_restructure.sh @@ -8,7 +8,7 @@ fi . ./util.sh # HDFS restructure version -JAR_VERSION=0.3.1 +JAR_VERSION=0.3.2 # HDFS restructure JAR JAR="restructurehdfs-all-${JAR_VERSION}.jar" From 0f68406f6561b76e256b2c40c6f49c77e1886042 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Tue, 8 May 2018 15:14:55 +0100 Subject: [PATCH 34/37] Remove stale docker image --- images/radar-mongodb-connector/Dockerfile | 27 -------- images/radar-mongodb-connector/README.md | 61 ------------------ .../radar-mongodb-connector/kafka_status.sh | 63 ------------------- images/radar-mongodb-connector/launch | 51 --------------- .../sink-mongodb.properties | 25 -------- 5 files changed, 227 deletions(-) delete mode 100644 images/radar-mongodb-connector/Dockerfile delete mode 100644 images/radar-mongodb-connector/README.md delete mode 100755 images/radar-mongodb-connector/kafka_status.sh delete mode 100755 images/radar-mongodb-connector/launch delete mode 100644 images/radar-mongodb-connector/sink-mongodb.properties diff --git a/images/radar-mongodb-connector/Dockerfile b/images/radar-mongodb-connector/Dockerfile deleted file mode 100644 index 74f9faca6..000000000 --- a/images/radar-mongodb-connector/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -FROM confluentinc/cp-kafka-connect:3.2.1 - -MAINTAINER Nivethika M , Joris B , Francesco N - -LABEL description="RADAR-base Backend- MongoDB Sink Connector" - -# Deploy RADAR-Backend - Mongodb Sink Connector -ADD https://github.com/RADAR-base/RADAR-MongoDB-Sink-Connector/releases/download/0.1/radar-mongodb-sink-connector-0.1.jar /etc/kafka-connect/jars/ - -# Load topics validator -COPY ./kafka_status.sh /home/kafka_status.sh - -# Load modified launcher -COPY launch /etc/confluent/docker/launch diff --git a/images/radar-mongodb-connector/README.md b/images/radar-mongodb-connector/README.md deleted file mode 100644 index db7ee90db..000000000 --- a/images/radar-mongodb-connector/README.md +++ /dev/null @@ -1,61 +0,0 @@ -# Dockerised RADAR-MongoDB-Connector - -It runs the RADAR-base MongoDB Connector project based on Confluent Platform 3.1.2, for more details check the [repository](https://github.com/RADAR-base/RADAR-MongoDbConnector). - -Create the docker image: -``` -$ docker build -t radarcns/radar-mongodb-connector ./ -``` - -Or pull from dockerhub: -``` -$ docker pull radarcns/radar-mongodb-connector:0.1 -``` - -## Configuration - -This image has to be extended with a volume with appropriate `sink.properties` - -Sample MongoDB sink.properties -``` -# Kafka consumer configuration -name=radar-connector-mongodb-sink - -# Kafka connector configuration -connector.class=org.radarcns.mongodb.MongoDbSinkConnector -tasks.max=1 - -# Topics that will be consumed -topics=topic1, topic2 - -# MongoDB server -mongo.host=mongo -mongo.port=27017 - -# MongoDB configuration -mongo.username= -mongo.password= -mongo.database=mydbase - -# Collection name for putting data into the MongoDB database. The {$topic} token will be replaced -# by the Kafka topic name. -#mongo.collection.format={$topic} - -# Factory class to do the actual record conversion -record.converter.class=org.radarcns.sink.mongodb.RecordConverterFactoryRadar -``` - -## Runtime environment variables - -This container requires two environment variable: - -- `KAFKA_REST_PROXY`: a valid Rest-Proxy instance -- `TOPIC_LIST`: a comma separated list containing all required topic names - -Before starting the streams, it waits until all topics inside TOPIC_LIST are available. This check is performed using the /topic Rest-Proxy API, for more details click here. - -Note that connector's REST_PORT must be different from the one used by Rest-Proxy. - -## How to run - -For a complete use case scenario, check the RADAR-base `docker-compose` file available [here](https://github.com/RADAR-base/RADAR-Docker/blob/backend-integration/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml) diff --git a/images/radar-mongodb-connector/kafka_status.sh b/images/radar-mongodb-connector/kafka_status.sh deleted file mode 100755 index e488b1eef..000000000 --- a/images/radar-mongodb-connector/kafka_status.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -# Check if variables exist -if [ -z "$CONNECT_ZOOKEEPER_CONNECT" ]; then - echo "CONNECT_ZOOKEEPER_CONNECT is not defined" - exit 2 -fi - -if [ -z "$CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL" ]; then - echo "CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL is not defined" - exit 4 -fi - -KAFKA_BROKERS=${KAFKA_BROKERS:-3} - -max_timeout=32 - -tries=10 -timeout=1 -while true; do - ZOOKEEPER_CHECK=$(zookeeper-shell $CONNECT_ZOOKEEPER_CONNECT <<< "ls /brokers/ids") - ZOOKEEPER_CHECK="${ZOOKEEPER_CHECK##*$'\n'}" - ZOOKEEPER_CHECK="$(echo -e "${ZOOKEEPER_CHECK}" | tr -d '[:space:]' | tr -d '[' | tr -d ']')" - - IFS=',' read -r -a array <<< $ZOOKEEPER_CHECK - LENGTH=${#array[@]} - if [ "$LENGTH" -eq "$KAFKA_BROKERS" ]; then - echo "Kafka brokers available." - break - fi - - tries=$((tries - 1)) - if [ $tries -eq 0 ]; then - echo "FAILED: KAFKA BROKERs NOT READY." - exit 5 - fi - echo "Expected $KAFKA_BROKERS brokers but found only $LENGTH. Waiting $timeout second before retrying ..." - sleep $timeout - if [ $timeout -lt $max_timeout ]; then - timeout=$((timeout * 2)) - fi -done - -tries=10 -timeout=1 -while true; do - if wget --spider -q "${CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL}/subjects" 2>/dev/null; then - break - fi - tries=$((tries - 1)) - if [ $tries -eq 0 ]; then - echo "FAILED TO REACH SCHEMA REGISTRY." - exit 6 - fi - echo "Failed to reach schema registry. Retrying in ${timeout} seconds." - sleep $timeout - if [ $timeout -lt $max_timeout ]; then - timeout=$((timeout * 2)) - fi -done - - -echo "Kafka is available. Ready to go!" diff --git a/images/radar-mongodb-connector/launch b/images/radar-mongodb-connector/launch deleted file mode 100755 index 5b584b6f2..000000000 --- a/images/radar-mongodb-connector/launch +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2016 Confluent Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Override this section from the script to include the com.sun.management.jmxremote.rmi.port property. -if [ -z "$KAFKA_JMX_OPTS" ]; then - export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " -fi - -# The JMX client needs to be able to connect to java.rmi.server.hostname. -# The default for bridged n/w is the bridged IP so you will only be able to connect from another docker container. -# For host n/w, this is the IP that the hostname on the host resolves to. - -# If you have more that one n/w configured, hostname -i gives you all the IPs, -# the default is to pick the first IP (or network). -export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)} - -if [ "$KAFKA_JMX_PORT" ]; then - # This ensures that the "if" section for JMX_PORT in kafka launch script does not trigger. - export JMX_PORT=$KAFKA_JMX_PORT - export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" -fi - -# Busy waiting loop that waits untill all topic are available -echo "===> Wait for infrastructure ..." -./home/kafka_status.sh -radar_check=$? -if [ "$radar_check" -ne 0 ]; then - exit $radar_check -fi - -echo "===> Launching ${COMPONENT} ... new" -# Add external jars to the classpath -# And this also makes sure that the CLASSPATH does not start with ":/etc/..." -# because this causes the plugin scanner to scan the entire disk. -export CLASSPATH="/etc/${COMPONENT}/jars/*" -echo $CLASSPATH -exec connect-standalone /etc/"${COMPONENT}"/"${COMPONENT}".properties /etc/"${COMPONENT}"/sink.properties diff --git a/images/radar-mongodb-connector/sink-mongodb.properties b/images/radar-mongodb-connector/sink-mongodb.properties deleted file mode 100644 index f4b641e20..000000000 --- a/images/radar-mongodb-connector/sink-mongodb.properties +++ /dev/null @@ -1,25 +0,0 @@ -# Kafka consumer configuration -name=radar-connector-mongodb-sink - -# Kafka connector configuration -connector.class=org.radarcns.mongodb.MongoDbSinkConnector -tasks.max=1 - -# Topics that will be consumed -topics=android_empatica_e4_battery_level,android_empatica_e4_battery_level_output - -# MongoDB server -mongo.host=mongo -mongo.port=27017 - -# MongoDB configuration -#mongo.username= -#mongo.password= -mongo.database=mydbase - -# Collection name for putting data into the MongoDB database. The {$topic} token will be replaced -# by the Kafka topic name. -#mongo.collection.format={$topic} - -# Factory class to do the actual record conversion -record.converter.class=org.radarcns.sink.mongodb.RecordConverterFactoryRadar From 14b48f9edcb1865c0ca348069a1308e85ffd3a46 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Tue, 8 May 2018 17:57:33 +0100 Subject: [PATCH 35/37] Production style multinode zookeeper cluster --- .../radar-cp-hadoop-stack/docker-compose.yml | 73 ++++++++++++++++--- 1 file changed, 64 insertions(+), 9 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 0ae5d9514..915bea556 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -30,6 +30,12 @@ volumes: kafka-1-data: {} kafka-2-data: {} kafka-3-data: {} + zookeeper-1-data: {} + zookeeper-2-data: {} + zookeeper-3-data: {} + zookeeper-1-log: {} + zookeeper-2-log: {} + zookeeper-3-log: {} radar-backend-monitor-disconnect-data: {} certs: external: true @@ -45,8 +51,8 @@ services: networks: - zookeeper volumes: - - /var/lib/zookeeper/data - - /var/lib/zookeeper/logs + - zookeeper-1-data:/var/lib/zookeeper/data + - zookeeper-1-log:/var/lib/zookeeper/logs restart: always environment: ZOOKEEPER_SERVER_ID: 1 @@ -54,7 +60,50 @@ services: ZOOKEEPER_TICK_TIME: 2000 ZOOKEEPER_INIT_LIMIT: 5 ZOOKEEPER_SYNC_LIMIT: 2 - ZOOKEEPER_SERVERS: zookeeper-1:2888:3888 + ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888 + healthcheck: + test: ["CMD", "/bin/bash", "-c", "[ $$(echo dump | nc zookeeper-1 2181 | head -c1 | wc -c) -gt 0 ] || exit 1"] + interval: 1m30s + timeout: 5s + retries: 3 + + zookeeper-2: + image: confluentinc/cp-zookeeper:4.1.0 + networks: + - zookeeper + volumes: + - zookeeper-2-data:/var/lib/zookeeper/data + - zookeeper-2-log:/var/lib/zookeeper/logs + restart: always + environment: + ZOOKEEPER_SERVER_ID: 2 + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + ZOOKEEPER_INIT_LIMIT: 5 + ZOOKEEPER_SYNC_LIMIT: 2 + ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888 + healthcheck: + test: ["CMD", "/bin/bash", "-c", "[ $$(echo dump | nc zookeeper-1 2181 | head -c1 | wc -c) -gt 0 ] || exit 1"] + interval: 1m30s + timeout: 5s + retries: 3 + + + zookeeper-3: + image: confluentinc/cp-zookeeper:4.1.0 + networks: + - zookeeper + volumes: + - zookeeper-3-data:/var/lib/zookeeper/data + - zookeeper-3-log:/var/lib/zookeeper/logs + restart: always + environment: + ZOOKEEPER_SERVER_ID: 3 + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + ZOOKEEPER_INIT_LIMIT: 5 + ZOOKEEPER_SYNC_LIMIT: 2 + ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888 healthcheck: test: ["CMD", "/bin/bash", "-c", "[ $$(echo dump | nc zookeeper-1 2181 | head -c1 | wc -c) -gt 0 ] || exit 1"] interval: 1m30s @@ -73,10 +122,12 @@ services: - kafka-1-data:/var/lib/kafka/data depends_on: - zookeeper-1 + - zookeeper-2 + - zookeeper-3 restart: always environment: KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181 + KAFKA_ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181" KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_LOG_RETENTION_HOURS: 730 @@ -96,11 +147,13 @@ services: volumes: - kafka-2-data:/var/lib/kafka/data depends_on: - - kafka-1 + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 restart: always environment: KAFKA_BROKER_ID: 2 - KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181 + KAFKA_ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181" KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:9092 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_LOG_RETENTION_HOURS: 730 @@ -120,11 +173,13 @@ services: volumes: - kafka-3-data:/var/lib/kafka/data depends_on: - - kafka-2 + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 restart: always environment: KAFKA_BROKER_ID: 3 - KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181 + KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:9092 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_LOG_RETENTION_HOURS: 730 @@ -640,7 +695,7 @@ services: depends_on: - zookeeper-1 environment: - ZK_HOSTS: zookeeper-1:2181 + ZK_HOSTS: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 restart: always healthcheck: test: ["CMD-SHELL", "[ $$(wget -q -O - localhost:9000/kafkamanager/api/health) = healthy ] || exit 1"] From f756ef88693940366b146f5f053b60bf0b5e6cbb Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Tue, 8 May 2018 18:04:55 +0100 Subject: [PATCH 36/37] Better kafka manager dependencies --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 915bea556..118ec1336 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -694,6 +694,9 @@ services: - api depends_on: - zookeeper-1 + - kafka-1 + - kafka-2 + - kafka-3 environment: ZK_HOSTS: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 restart: always From b9805557ed18e24f566ff5df8a0d69f4d31fd1c1 Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Wed, 9 May 2018 09:49:36 +0100 Subject: [PATCH 37/37] Update README.md --- dcompose-stack/radar-cp-hadoop-stack/README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index cab469561..32bc096e4 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -5,6 +5,8 @@ This docker-compose stack contains the full operational RADAR platform. Once con ## Configuration 1. First copy `etc/env.template` file to `./.env` and check and modify all its variables. + + 1.1. To have a valid HTTPS connection for a public host, set `SELF_SIGNED_CERT=no`. You need to provide a public valid DNS name as `SERVER_NAME` for SSL certificate to work. IP addresses will not work. 1.2. Set `MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET` to a secret to be used by the Management Portal frontend. @@ -17,7 +19,7 @@ This docker-compose stack contains the full operational RADAR platform. Once con 4. Copy `etc/managementportal/config/oauth_client_details.csv.template` to `etc/managementportal/config/oauth_client_details.csv` and change OAuth client credentials for production MP. The OAuth client for the frontend will be loaded automatically and does not need to be listed in this file. This file will be read at each startup. The current implementation overwrites existing clients with the same client ID, so be aware of this if you have made changes to a client listed in this file using the Management Portal frontend. This behaviour might change in the future. -5. Finally, copy `etc/radar.yml.template` to `etc/radar.yml` and edit it, especially concerning the monitor email address configuration. +5. Finally, copy `etc/radar-backend/radar.yml.template` to `etc/radar-backend/radar.yml` and edit it, especially concerning the monitor email address configuration. 6. (Optional) Note: To have different flush.size for different topics, you can create multipe property configurations for a single connector. To do that, @@ -104,7 +106,7 @@ CSV-structured data can be gotten from HDFS by running ```shell ./hdfs_restructure.sh /topicAndroidNew ``` -This will put all CSV files in the destination directory, with subdirectory structure `PatientId/SensorType/Date_Hour.csv`. +This will put all CSV files in the destination directory, with subdirectory structure `ProjectId/SubjectId/SensorType/Date_Hour.csv`. ## Cerificate @@ -128,7 +130,7 @@ Portainer provides simple interactive UI-based docker management. If running loc ### Kafka Manager -The [kafka-manager](https://github.com/yahoo/kafka-manager) is an interactive web based tool for managing Apache Kafka. Kafka manager has beed integrated in the stack. It is accessible at +The [kafka-manager](https://github.com/yahoo/kafka-manager) is an interactive web based tool for managing Apache Kafka. Kafka manager has beed integrated in the stack. It is accessible at `http:///kafkamanager/` ### Check Health Each of the containers in the stack monitor their own health and show the output as healthy or unhealthy. A script called check-health.sh is used to check this output and send an email to the maintainer if a container is unhealthy. @@ -137,13 +139,13 @@ First check that the `MAINTAINER_EMAIL` in the .env file is correct. Then make sure that the SMTP server is configured properly and running. -If systemd integration is enabled, the check-health.sh script will check health of containers every five minutes. It can then be run directly by running +If systemd integration is enabled, the check-health.sh script will check health of containers every five minutes. It can then be run directly by running if systemd wrappers have been installed ``` sudo systemctl start radar-check-health.service ``` Otherwise, the following manual commands can be invoked. -Then just add a cron job to run the `check-health.sh` script periodically like - +Add a cron job to run the `check-health.sh` script periodically like - 1. Edit the crontab file for the current user by typing `$ crontab -e` 2. Add your job and time interval. For example, add the following for checking health every 5 mins -