diff --git a/.travis.yml b/.travis.yml index fd97af7d2..42b2bec3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ script: - sudo docker volume create certs - sudo docker volume create certs-data - cp etc/radar.yml.template etc/radar.yml - - cp etc/nginx.conf.template etc/nginx.conf + - cp etc/webserver/nginx.conf.template etc/webserver/nginx.conf - cp etc/sink-hdfs.properties.template etc/sink-hdfs.properties - cp etc/sink-mongo.properties.template etc/sink-mongo.properties - sudo $HOME/bin/docker-compose up -d --build && sleep 15 && [ -z "$($HOME/bin/docker-compose ps | tail -n +3 | grep " Exit ")" ] diff --git a/README.md b/README.md index b512e7e11..a3ba7a5bd 100644 --- a/README.md +++ b/README.md @@ -65,113 +65,19 @@ In addition to Confluent Kafka platform components, RADAR-CNS platform offers * RADAR-Monitor - Status monitors, * [RADAR-HotStorage](https://github.com/RADAR-CNS/RADAR-HotStorage) via MongoDB, * [RADAR-REST API](https://github.com/RADAR-CNS/RADAR-RestApi), -* a Hadoop cluster, and -* an email server. - +* A Hadoop cluster, and +* An email server. +* Management Portal - A web portal to manage patient monitoring studies. +* RADAR-Gateway - A validating gateway to allow only valid and authentic data to the platform +* Catalog server - A Service to share source-types configured in the platform. To run RADAR-CNS stack in a single node setup: -1. Navigate to `radar-hadoop-cp-stack`: +1. Navigate to `radar-cp-hadoop-stack`: ```shell - cd RADAR-Docker/dcompose-stack/radar-hadoop-cp-stack/ - ``` -2. Configure monitor settings in `radar.yml`: - - ```yaml - battery_monitor: - level: CRITICAL - email_address: - - notify-1@example.com - - notify-2@example.com - email_host: smtp - email_port: 25 - email_user: user@example.com - topics: - - android_empatica_e4_battery_level - - disconnect_monitor: - # timeout in milliseconds -> 5 minutes - timeout: 300000 - email_address: - - notify-1@example.com - - notify-2@example.com - email_host: smtp - email_port: 25 - email_user: user@example.com - # temperature readings are sent very regularly, but - # not too often. - topics: - - android_empatica_e4_temperature - ``` -3. Create `smtp.env` and configure your email settings following `smtp.env.template`. Configure alternative mail providers like Amazon SES or Gmail by using the parameters of the [`namshi/smtp` Docker image](https://hub.docker.com/r/namshi/smtp/). -4. (Optional) Modify flush.size and HDFS direcotory for Cold storage in `sink-hdfs.properties` - - ```ini - flush.size= - topics.dir=/path/to/data - ``` - Note: To have different flush.size for different topics, you can create multipe property configurations for a single connector. To do that, - - 4.1 Create multipe property files that have different `flush.size` for given topics. - Examples [sink-hdfs-high.properties](https://github.com/RADAR-CNS/RADAR-Docker/blob/dev/dcompose-stack/radar-cp-hadoop-stack/sink-hdfs-high.properties) , [sink-hdfs-low.properties](https://github.com/RADAR-CNS/RADAR-Docker/blob/dev/dcompose-stack/radar-cp-hadoop-stack/sink-hdfs-low.properties) - - 4.2 Add `CONNECTOR_PROPERTY_FILE_PREFIX: ` enviornment variable to `radar-hdfs-connector` service in `docker-compose` file. - - 4.3 Add created property files to the `radar-hdfs-connector` service in `docker-compose` with name abides to prefix-value mentioned in `CONNECTOR_PROPERTY_FILE_PREFIX` - - ```ini - radar-hdfs-connector: - image: radarcns/radar-hdfs-connector-auto:0.2 - restart: on-failure - volumes: - - ./sink-hdfs-high.properties:/etc/kafka-connect/sink-hdfs-high.properties - - ./sink-hdfs-low.properties:/etc/kafka-connect/sink-hdfs-low.properties - environment: - CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092 - CONNECTOR_PROPERTY_FILE_PREFIX: "sink-hdfs" - ``` - -5. Configure Hot Storage settings in `.env` file - - ```ini - HOTSTORAGE_USERNAME=mongodb-user - HOTSTORAGE_PASSWORD=XXXXXXXX - HOTSTORAGE_NAME=mongodb-database - ``` -6. To install the stack - - ```shell - sudo ./install-radar-stack.sh + cd RADAR-Docker/dcompose-stack/radar-cp-hadoop-stack/ ``` - -To stop RADAR-CNS stack on a single node setup, run - -```shell -cd RADAR-Docker/dcompose-stack/radar-hadoop-cp-stack/ -sudo ./stop-radar-stack.sh -``` -To reboot RADAR-CNS stack on a single node setup, run - -```shell -cd RADAR-Docker/dcompose-stack/radar-hadoop-cp-stack/ -sudo ./reboot-radar-stack.sh -``` -To start RADAR-CNS stack on a single node setup after installing, run - -```shell -cd RADAR-Docker/dcompose-stack/radar-hadoop-cp-stack/ -sudo ./start-radar-stack.sh -``` - -#### cAdvisor - -cAdvisor (Container Advisor) provides container users an understanding of the resource usage and performance characteristics of their running containers. - -To view current resource performance,if running locally, try [http://localhost:8181](http://localhost:8181). This will bring up the built-in Web UI. Clicking on `/docker` in `Subcontainers` takes you to a new window with all of the Docker containers listed individually. - -#### Portainer - -Portainer provides simple interactive UI-based docker management. If running locally, try [http://localhost:8182](http://localhost:8182) for portainer's UI. To set-up portainer follow this [link](https://www.ostechnix.com/portainer-an-easiest-way-to-manage-docker/). +2. Follow the README instructions there for correct configuration. ### Logging diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index ea25d5396..2d9d46b61 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -1,17 +1,44 @@ -# RADAR-CNS with a HDFS connector +# RADAR platform + +This docker-compose stack contains the full operational RADAR platform. Once configured, it is meant to run on a single server with at least 16 GB memory and 4 CPU cores. It is tested on Ubuntu 16.04 and on macOS 11.1 with Docker 17.06. ## Configuration -1. First move `etc/env.template` file to `./.env` and check and modify all its variables. To have a valid HTTPS connection for a public host, set `SELF_SIGNED_CERT=no`. You need to provide a public valid DNS name as `SERVER_NAME` for SSL certificate to work. IP addresses will not work. +1. First copy `etc/env.template` file to `./.env` and check and modify all its variables. + 1.1. To have a valid HTTPS connection for a public host, set `SELF_SIGNED_CERT=no`. You need to provide a public valid DNS name as `SERVER_NAME` for SSL certificate to work. IP addresses will not work. + + 1.2. Set `MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET` to a secret to be used by the Management Portal frontend. + + 1.3. If you want to enable auto import of source types from the catalog server set the variable `MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT` to `true`. + +2. Copy `etc/smtp.env.template` to `etc/smtp.env` and configure your email settings. Configure alternative mail providers like Amazon SES or Gmail by using the parameters of the [`namshi/smtp` Docker image](https://hub.docker.com/r/namshi/smtp/). + +3. Copy `etc/redcap-integration/radar.yml.template` to `etc/redcap-integration/radar.yml` and modify it to configure the properties of Redcap instance and the management portal. For reference on configuration of this file look at the Readme file here - . In the REDcap portal under Project Setup, define the Data Trigger as `https:///redcapint/trigger` + +4. Copy `etc/managementportal/config/oauth_client_details.csv.template` to `etc/managementportal/config/oauth_client_details.csv` and change OAuth client credentials for production MP. The OAuth client for the frontend will be loaded automatically and does not need to be listed in this file. This file will be read at each startup. The current implementation overwrites existing clients with the same client ID, so be aware of this if you have made changes to a client listed in this file using the Management Portal frontend. This behaviour might change in the future. + +5. Finally, copy `etc/radar.yml.template` to `etc/radar.yml` and edit it, especially concerning the monitor email address configuration. + +6. (Optional) Note: To have different flush.size for different topics, you can create multipe property configurations for a single connector. To do that, -2. Modify `etc/smtp.env.template` to set a SMTP host to send emails with, and move it to `etc/smtp.env`. The configuration settings are passed to a [namshi/smtp](https://hub.docker.com/r/namshi/smtp/) Docker container. This container supports a.o. regular SMTP and GMail. + 6.1 Create multipe property files that have different `flush.size` for given topics. + Examples [sink-hdfs-high.properties](https://github.com/RADAR-CNS/RADAR-Docker/blob/dev/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-high.properties) , [sink-hdfs-low.properties](https://github.com/RADAR-CNS/RADAR-Docker/blob/dev/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-low.properties) -3. Modify the `etc/redcap-integration/radar.yml.template` to configure the properties of Redcap instance and the management portal, and move it to `etc/redcap-integration/radar.yml`. For reference on configuration of this file look at the Readme file here - https://github.com/RADAR-CNS/RADAR-RedcapIntegration#configuration -In the REDcap portal under Project Setup, define the Data Trigger as `https:///redcapint/trigger` + 6.2 Add `CONNECTOR_PROPERTY_FILE_PREFIX: ` environment variable to `radar-hdfs-connector` service in `docker-compose` file. -4. Move `etc/managementportal/changelogs/config/liquibase/oauth_client_details.csv.template` to `etc/managementportal/changelogs/config/liquibase/oauth_client_details.csv` and change OAuth client credentials for production MP. (Except ManagementPortalapp) + 6.3 Add created property files to the `radar-hdfs-connector` service in `docker-compose` with name abides to prefix-value mentioned in `CONNECTOR_PROPERTY_FILE_PREFIX` -5. Finally, move `etc/radar.yml.template` to `etc/radar.yml` and edit it, especially concerning the monitor email address configuration. + ```ini + radar-hdfs-connector: + image: radarcns/radar-hdfs-connector-auto:0.2 + restart: on-failure + volumes: + - ./sink-hdfs-high.properties:/etc/kafka-connect/sink-hdfs-high.properties + - ./sink-hdfs-low.properties:/etc/kafka-connect/sink-hdfs-low.properties + environment: + CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092 + CONNECTOR_PROPERTY_FILE_PREFIX: "sink-hdfs" + ``` ## Usage @@ -21,6 +48,50 @@ Run ``` to start all the RADAR services. Use the `(start|stop|reboot)-radar-stack.sh` to start, stop or reboot it. Note: whenever `.env` or `docker-compose.yml` are modified, this script needs to be called again. To start a reduced set of containers, call `install-radar-stack.sh` with the intended containers as arguments. +To enable a `systemd` service to control the platform, run +```shell +./install-systemd-wrappers.sh +``` +After that command, the RADAR platform should be controlled via `systemctl`. +```shell +# query the latest status and logs +sudo systemctl status radar-docker + +# Stop radar-docker +sudo systemctl stop radar-docker + +# Restart all containers +sudo systemctl reload radar-docker + +# Start radar-docker +sudo systemctl start radar-docker + +# Full radar-docker system logs +sudo journalctl -u radar-docker +``` +The control scripts in this directory should preferably not be used if `systemctl` is used. To remove `systemctl` integration, run +``` +sudo systemctl disable radar-docker +sudo systemctl disable radar-output +sudo systemctl disable radar-check-health +sudo systemctl disable radar-renew-certificate +``` + +To clear all data from the platform, run +``` +sudo systemctl stop radar-docker +./docker-prune.sh +sudo systemctl start radar-docker +``` + +## Data extraction + +If systemd integration is enabled, HDFS data will be extracted to the `./output` directory every hour. It can then be run directly by running +``` +sudo systemctl start radar-output.service +``` +Otherwise, the following manual commands can be invoked. + Raw data can be extracted from this setup by running: ```shell @@ -35,4 +106,52 @@ CSV-structured data can be gotten from HDFS by running ``` This will put all CSV files in the destination directory, with subdirectory structure `PatientId/SensorType/Date_Hour.csv`. +## Cerificate + +If systemd integration is enabled, the ssl certificate will be renewed daily. It can then be run directly by running +``` +sudo systemctl start radar-renew-certificate.service +``` +Otherwise, the following manual commands can be invoked. If `SELF_SIGNED_CERT=no` in `./.env`, be sure to run `./renew_ssl_certificate.sh` daily to ensure that your certificate does not expire. + + +### cAdvisor + +cAdvisor (Container Advisor) provides container users an understanding of the resource usage and performance characteristics of their running containers. + +To view current resource performance,if running locally, try . This will bring up the built-in Web UI. Clicking on `/docker` in `Subcontainers` takes you to a new window with all of the Docker containers listed individually. + +### Portainer + +Portainer provides simple interactive UI-based docker management. If running locally, try for portainer's UI. To set-up portainer follow this [link](https://www.ostechnix.com/portainer-an-easiest-way-to-manage-docker/). + +### Kafka Manager + +The [kafka-manager](https://github.com/yahoo/kafka-manager) is an interactive web based tool for managing Apache Kafka. Kafka manager has beed integrated in the stack. It is accessible at + +### Check Health +Each of the containers in the stack monitor their own health and show the output as healthy or unhealthy. A script called check-health.sh is used to check this output and send an email to the maintainer if a container is unhealthy. + +First check that the `MAINTAINER_EMAIL` in the .env file is correct. + +Then make sure that the SMTP server is configured properly and running. + +If systemd integration is enabled, the check-health.sh script will check health of containers every five minutes. It can then be run directly by running +``` +sudo systemctl start radar-check-health.service +``` +Otherwise, the following manual commands can be invoked. + +Then just add a cron job to run the `check-health.sh` script periodically like - +1. Edit the crontab file for the current user by typing `$ crontab -e` +2. Add your job and time interval. For example, add the following for checking health every 5 mins - + +```*/5 * * * * /home/ubuntu/RADAR-Docker/dcompose-stack/radar-cp-hadoop-stack/check-health.sh``` + +You can check the logs of CRON by typing `$ grep CRON /var/log/syslog` +Also you will need to change the directory. So just add the following to the top of the check-health.sh script - +```sh +cd "$( dirname "${BASH_SOURCE[0]}" )" +``` + diff --git a/dcompose-stack/radar-cp-hadoop-stack/check-health.sh b/dcompose-stack/radar-cp-hadoop-stack/check-health.sh new file mode 100755 index 000000000..43e7156d3 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/check-health.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Check whether services are healthy. If not, restart them and notify the maintainer. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +. "$DIR/util.sh" +. .env + +function hipchat_notify() { + # Send notification via HipChat, if configured. + if [ "$HEALTHCHECK_HIPCHAT_NOTIFY" == "yes" ] ; then + if [ -z "$HEALTHCHECK_HIPCHAT_ROOM_ID" ] ; then + echo "Error: HipChat notifications are enabled, but \$HEALTHCHECK_HIPCHAT_ROOM_ID is undefined. Unable to send HipChat notification." + exit 1 + fi + + if [ -z "$HEALTHCHECK_HIPCHAT_TOKEN" ] ; then + echo "Error: HipChat notifications are enabled, but \$HEALTHCHECK_HIPCHAT_TOKEN is undefined. Unable to send HipChat notification." + exit 1 + fi + + color=$1 + body=$2 + curl -X POST -H "Content-Type: application/json" --header "Authorization: Bearer $HEALTHCHECK_HIPCHAT_TOKEN" \ + -d "{\"color\": \"$color\", \"message_format\": \"text\", \"message\": \"$body\" }" \ + https://api.hipchat.com/v2/room/$HEALTHCHECK_HIPCHAT_ROOM_ID/notification + fi +} + +unhealthy=() + +# get all human-readable service names +# see last line of loop +while read service; do + # check if a container was started for the service + container=$(sudo-linux docker-compose ps -q $service) + if [ -z "${container}" ]; then + # no container means no running service + continue + fi + health=$(sudo-linux docker inspect --format '{{.State.Health.Status}}' $container 2>/dev/null || echo "null") + if [ "$health" = "unhealthy" ]; then + echo "Service $service is unhealthy. Restarting." + unhealthy+=("${service}") + sudo-linux docker-compose restart ${service} + fi +done <<< "$(sudo-linux docker-compose config --services)" + +if [ "${#unhealthy[@]}" -eq 0 ]; then + if [ -f .unhealthy ]; then + rm -f .unhealthy + hipchat_notify green "All services are healthy again" + fi + echo "All services are healthy" +else + echo "$unhealthy services were unhealthy and have been restarted." + + # Send notification to MAINTAINER + # start up the mail container if not already started + sudo-linux docker-compose up -d smtp + # save the container, so that we can use exec to send an email later + container=$(sudo-linux docker-compose ps -q smtp) + SAVEIFS=$IFS + IFS=, + display_services="[${unhealthy[*]}]" + IFS=$SAVEIFS + display_host="${SERVER_NAME} ($(hostname -f), $(curl -s http://ipecho.net/plain))" + body="Services on $display_host are unhealthy. Services $display_services have been restarted. Please log in for further information." + echo "Sent notification to $MAINTAINER_EMAIL" + echo "$body" | sudo-linux docker exec -i ${container} mail -aFrom:$FROM_EMAIL "-s[RADAR] Services on ${SERVER_NAME} unhealthy" $MAINTAINER_EMAIL + + echo "${unhealthy[@]}" > .unhealthy + + hipchat_notify red "$body" + + exit 1 +fi diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 0810b538d..a2aae2413 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -22,9 +22,6 @@ networks: hotstorage: driver: bridge internal: true - redcap: - driver: bridge - internal: true management: driver: bridge internal: true @@ -33,18 +30,18 @@ volumes: kafka-1-data: {} kafka-2-data: {} kafka-3-data: {} + radar-backend-monitor-disconnect-data: {} certs: external: true certs-data: external: true services: - #---------------------------------------------------------------------------# # Zookeeper Cluster # #---------------------------------------------------------------------------# zookeeper-1: - image: confluentinc/cp-zookeeper:3.1.2-1 + image: confluentinc/cp-zookeeper:3.2.1-5 networks: - zookeeper volumes: @@ -58,12 +55,17 @@ services: ZOOKEEPER_INIT_LIMIT: 5 ZOOKEEPER_SYNC_LIMIT: 2 ZOOKEEPER_SERVERS: zookeeper-1:2888:3888 + healthcheck: + test: ["CMD", "/bin/bash", "-c", "[ $$(echo dump | nc zookeeper-1 2181 | head -c1 | wc -c) -gt 0 ] || exit 1"] + interval: 1m30s + timeout: 5s + retries: 3 #---------------------------------------------------------------------------# # Kafka Cluster # #---------------------------------------------------------------------------# kafka-1: - image: confluentinc/cp-kafka:3.1.2-1 + image: confluentinc/cp-kafka:3.2.1-5 networks: - kafka - zookeeper @@ -78,9 +80,15 @@ services: KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_LOG_RETENTION_HOURS: 730 + KAFKA_LOG4J_LOGGERS: kafka.producer.async.DefaultEventHandler=INFO,kafka.controller=INFO,state.change.logger=INFO + healthcheck: + test: ["CMD-SHELL", "echo dump | nc zookeeper-1 2181 | grep -q /brokers/ids/1 || exit 1"] + interval: 1m30s + timeout: 10s + retries: 3 kafka-2: - image: confluentinc/cp-kafka:3.1.2-1 + image: confluentinc/cp-kafka:3.2.1-5 networks: - kafka - zookeeper @@ -95,9 +103,15 @@ services: KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:9092 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_LOG_RETENTION_HOURS: 730 + KAFKA_LOG4J_LOGGERS: kafka.producer.async.DefaultEventHandler=INFO,kafka.controller=INFO,state.change.logger=INFO + healthcheck: + test: ["CMD-SHELL", "echo dump | nc zookeeper-1 2181 | grep -q /brokers/ids/2 || exit 1"] + interval: 1m30s + timeout: 10s + retries: 3 kafka-3: - image: confluentinc/cp-kafka:3.1.2-1 + image: confluentinc/cp-kafka:3.2.1-5 networks: - kafka - zookeeper @@ -112,12 +126,18 @@ services: KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:9092 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_LOG_RETENTION_HOURS: 730 + KAFKA_LOG4J_LOGGERS: kafka.producer.async.DefaultEventHandler=INFO,kafka.controller=INFO,state.change.logger=INFO + healthcheck: + test: ["CMD-SHELL", "echo dump | nc zookeeper-1 2181 | grep -q /brokers/ids/3 || exit 1"] + interval: 1m30s + timeout: 10s + retries: 3 #---------------------------------------------------------------------------# # Schema Registry # #---------------------------------------------------------------------------# schema-registry-1: - image: confluentinc/cp-schema-registry:3.1.2-1 + image: confluentinc/cp-schema-registry:3.2.1-5 networks: - kafka - zookeeper @@ -131,16 +151,20 @@ services: SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper-1:2181 SCHEMA_REGISTRY_HOST_NAME: schema-registry-1 SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 + healthcheck: + test: ["CMD", "curl", "-f", "localhost:8081/subjects"] + interval: 1m30s + timeout: 5s + retries: 3 #---------------------------------------------------------------------------# # REST proxy # #---------------------------------------------------------------------------# rest-proxy-1: - image: confluentinc/cp-kafka-rest:3.1.2-1 + image: confluentinc/cp-kafka-rest:3.2.1-5 networks: - kafka - zookeeper - - api depends_on: - kafka-1 - kafka-2 @@ -152,13 +176,18 @@ services: KAFKA_REST_LISTENERS: http://0.0.0.0:8082 KAFKA_REST_SCHEMA_REGISTRY_URL: http://schema-registry-1:8081 KAFKA_REST_HOST_NAME: rest-proxy-1 + healthcheck: + test: ["CMD", "curl", "-f", "localhost:8082/topics"] + interval: 1m30s + timeout: 5s + retries: 3 #---------------------------------------------------------------------------# # Kafka Init # #---------------------------------------------------------------------------# kafka-init: build: kafka-radarinit - image: radarcns/kafka-init:3.1.2-1 + image: radarcns/kafka-init networks: - kafka - zookeeper @@ -167,14 +196,15 @@ services: - kafka-1 - kafka-2 - kafka-3 - - rest-proxy-1 + - schema-registry-1 + volumes: + - ./etc/schema:/schema/conf environment: KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181 - KAFKA_BROKERS: 3 - RADAR_TOPICS: ${RADAR_TOPIC_LIST} - RADAR_PARTITIONS: 3 - RADAR_REPLICATION_FACTOR: 3 - + KAFKA_SCHEMA_REGISTRY: http://schema-registry-1:8081 + KAFKA_NUM_BROKERS: 3 + RADAR_NUM_PARTITIONS: 3 + RADAR_NUM_REPLICATION_FACTOR: 3 #---------------------------------------------------------------------------# # RADAR Hot Storage # @@ -191,6 +221,11 @@ services: RADAR_USER: ${HOTSTORAGE_USERNAME} RADAR_PWD: ${HOTSTORAGE_PASSWORD} RADAR_DB: ${HOTSTORAGE_NAME} + healthcheck: + test: ["CMD", "mongo", "-u", "${HOTSTORAGE_USERNAME}", "-p", "${HOTSTORAGE_PASSWORD}", "${HOTSTORAGE_NAME}", "--eval", "db"] + interval: 1m + timeout: 5s + retries: 3 #---------------------------------------------------------------------------# # RADAR REST API # @@ -206,7 +241,7 @@ services: volumes: - "./etc/rest-api:/usr/local/tomcat/conf/radar" healthcheck: - test: ["CMD", "curl", "-IX", "HEAD", "http://localhost:8080/radar/api/"] + test: ["CMD", "curl", "-f", "http://localhost:8080/radar/api/source/specification/ANDROID"] interval: 1m timeout: 5s retries: 3 @@ -218,9 +253,8 @@ services: radar-integration: image: radarcns/radar-redcapintegration:latest networks: - - redcap + - api - default - - management depends_on: - managementportal-app restart: always @@ -264,6 +298,11 @@ services: environment: CORE_CONF_fs_defaultFS: hdfs://hdfs-namenode:8020 HDFS_CONF_dfs_replication: 2 + healthcheck: + test: ["CMD", "hdfs", "dfs", "-ls", "/"] + interval: 1m + timeout: 5s + retries: 3 hdfs-datanode-2: image: uhopper/hadoop-datanode:2.7.2 @@ -275,6 +314,11 @@ services: environment: CORE_CONF_fs_defaultFS: hdfs://hdfs-namenode:8020 HDFS_CONF_dfs_replication: 2 + healthcheck: + test: ["CMD", "hdfs", "dfs", "-ls", "/"] + interval: 1m + timeout: 5s + retries: 3 hdfs-namenode: build: hdfs-namenode @@ -288,6 +332,11 @@ services: environment: CLUSTER_NAME: radar-cns HDFS_CONF_dfs_namenode_name_dir: file:///hadoop/dfs/name/1,file:///hadoop/dfs/name/2 + healthcheck: + test: ["CMD", "hdfs", "dfs", "-ls", "/"] + interval: 1m + timeout: 5s + retries: 3 #---------------------------------------------------------------------------# # Email server # @@ -295,8 +344,8 @@ services: smtp: image: namshi/smtp:latest networks: - - default - mail + - default volumes: - /var/spool/exim restart: always @@ -307,7 +356,7 @@ services: # RADAR mongo connector # #---------------------------------------------------------------------------# radar-mongodb-connector: - image: radarcns/radar-mongodb-connector-auto:0.2.1 + image: radarcns/radar-mongodb-connector-auto:0.1 restart: on-failure volumes: - ./etc/sink-mongo.properties:/etc/kafka-connect/sink.properties @@ -344,20 +393,27 @@ services: CONNECT_CONSUMER_MAX_POLL_INTERVAL_MS: 300000 CONNECT_CONSUMER_SESSION_TIMEOUT_MS: 10000 CONNECT_CONSUMER_HEARTBEAT_INTERVAL_MS: 3000 - TOPIC_LIST: ${RADAR_TOPIC_LIST} + KAFKA_BROKERS: 3 + # For backwards compatibility + TOPIC_LIST: "application_record_counts" + healthcheck: + test: ["CMD-SHELL", "curl -sf localhost:8083/connectors/radar-connector-mongodb-sink/status | grep -o '\"state\":\"[^\"]*\"' | tr '\\n' ',' | grep -vq FAILED || exit 1"] + interval: 1m + timeout: 5s + retries: 3 + #---------------------------------------------------------------------------# # RADAR HDFS connector # #---------------------------------------------------------------------------# radar-hdfs-connector: - image: radarcns/radar-hdfs-connector-auto:0.2.1 + image: radarcns/radar-hdfs-connector-auto:0.1 restart: on-failure volumes: - ./etc/sink-hdfs.properties:/etc/kafka-connect/sink-hdfs.properties networks: - zookeeper - kafka - - api - hadoop depends_on: - zookeeper-1 @@ -386,20 +442,29 @@ services: CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/tmp/connect2.offset" CONNECT_REST_ADVERTISED_HOST_NAME: "radar-hdfs-connector" CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181 - TOPIC_LIST: ${RADAR_TOPIC_LIST} CONNECTOR_PROPERTY_FILE_PREFIX: "sink-hdfs" KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" + KAFKA_BROKERS: 3 + # For backwards compatibility + TOPIC_LIST: "application_record_counts" + healthcheck: + test: ["CMD-SHELL", "curl -sf localhost:8083/connectors/radar-hdfs-sink-android-15000/status | grep -o '\"state\":\"[^\"]*\"' | tr '\\n' ',' | grep -vq FAILED || exit 1"] + interval: 1m + timeout: 5s + retries: 3 #---------------------------------------------------------------------------# # RADAR backend streams # #---------------------------------------------------------------------------# radar-backend-stream: - image: radarcns/radar-backend-kafka-auto:0.2 + image: radarcns/radar-backend-kafka-auto:0.1.2 command: - stream networks: - zookeeper - kafka + # for getting the play store category + - default depends_on: - zookeeper-1 - kafka-1 @@ -412,15 +477,15 @@ services: restart: always environment: KAFKA_REST_PROXY: http://rest-proxy-1:8082 - TOPIC_LIST: ${RADAR_TOPIC_LIST} + KAFKA_SCHEMA_REGISTRY: http://schema-registry-1:8081 + KAFKA_BROKERS: 3 #---------------------------------------------------------------------------# # RADAR backend monitor # #---------------------------------------------------------------------------# radar-backend-monitor: - image: radarcns/radar-backend-kafka-auto:0.2 - command: - - monitor + image: radarcns/radar-backend-kafka-auto:0.1.2 + command: monitor networks: - zookeeper - kafka @@ -435,10 +500,14 @@ services: - smtp volumes: - ./etc/radar.yml:/etc/radar.yml + - radar-backend-monitor-disconnect-data:/var/lib/radar/data restart: always environment: KAFKA_REST_PROXY: http://rest-proxy-1:8082 - TOPIC_LIST: ${RADAR_TOPIC_LIST} + KAFKA_SCHEMA_REGISTRY: http://schema-registry-1:8081 + KAFKA_BROKERS: 3 + # For backwards compatibility + TOPIC_LIST: "application_record_counts" #---------------------------------------------------------------------------# # Docker Monitoring # @@ -469,11 +538,10 @@ services: # Webserver # #---------------------------------------------------------------------------# webserver: - image: nginx:1.12-alpine + image: nginx:1.13.1-alpine restart: always networks: - api - - redcap - monitor - default depends_on: @@ -481,27 +549,37 @@ services: - portainer - rest-api - schema-registry-1 - - rest-proxy-1 + - gateway - dashboard - radar-integration - managementportal-app + - kafka-manager ports: - "80:80" - "443:443" volumes: - certs:/etc/letsencrypt - certs-data:/data/letsencrypt - - "./etc/nginx.conf:/etc/nginx/nginx.conf:ro" + - "./etc/webserver/nginx.conf:/etc/nginx/nginx.conf:ro" + - "./etc/webserver/cors.conf:/etc/nginx/cors.conf:ro" + # healthcheck hard to do, however, it is possible to monitor this externally + # with + # docker logs --since 2m radarcphadoopstack_webserver_1 | grep "connect() failed" + #---------------------------------------------------------------------------# + # Management Portal # + #---------------------------------------------------------------------------# managementportal-app: - image: radarcns/management-portal:latest + image: radarcns/management-portal:0.3.1 networks: - default + - api - management - mail depends_on: - managementportal-postgresql - smtp + - catalog-server environment: SPRING_PROFILES_ACTIVE: prod,swagger SPRING_DATASOURCE_URL: jdbc:postgresql://managementportal-postgresql:5432/managementportal @@ -509,18 +587,97 @@ services: SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD} MANAGEMENTPORTAL_MAIL_FROM: ${FROM_EMAIL} MANAGEMENTPORTAL_MAIL_BASEURL: https://${SERVER_NAME}/managementportal + MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET: ${MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET} + MANAGEMENTPORTAL_OAUTH_CLIENTS_FILE: /mp-includes/config/oauth_client_details.csv + MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT: ${MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT} + MANAGEMENTPORTAL_CATALOGUE_SERVER_SERVER_URL: http://catalog-server:9010/source-types JHIPSTER_SLEEP: 10 # gives time for the database to boot before the application JAVA_OPTS: -Xmx256m # maximum heap size for the JVM running ManagementPortal, increase this as necessary volumes: - - ./etc/managementportal/changelogs:/changelogs + - ./etc/managementportal/:/mp-includes/ + healthcheck: + test: ["CMD", "wget", "--spider", "localhost:8080/managementportal/oauth/token_key"] + interval: 1m30s + timeout: 5s + retries: 3 + managementportal-postgresql: image: postgres:9.6.2 volumes: - - "${MP_POSTGRES_DIR}/:/var/lib/postgresql/" + - "${MP_POSTGRES_DIR}/data/:/var/lib/postgresql/data/" environment: POSTGRES_USER : ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_DB: managementportal networks: - - management \ No newline at end of file + - management + healthcheck: + test: ["CMD-SHELL", "PGPASSWORD='${POSTGRES_PASSWORD}' psql -U '${POSTGRES_USER}' managementportal -l || exit 1"] + interval: 1m30s + timeout: 5s + retries: 3 + + #---------------------------------------------------------------------------# + # Kafka Manager # + #---------------------------------------------------------------------------# + kafka-manager: + image: radarcns/kafka-manager + networks: + - zookeeper + - kafka + - api + depends_on: + - zookeeper-1 + environment: + ZK_HOSTS: zookeeper-1:2181 + restart: always + healthcheck: + test: ["CMD-SHELL", "[ $$(wget -q -O - localhost:9000/kafkamanager/api/health) = healthy ] || exit 1"] + interval: 1m30s + timeout: 5s + retries: 3 + + #---------------------------------------------------------------------------# + # RADAR Gateway # + #---------------------------------------------------------------------------# + gateway: + image: radarcns/radar-gateway:0.1.2 + networks: + - api + - kafka + depends_on: + - rest-proxy-1 + - managementportal-app + volumes: + - ./etc/gateway/radar-is.yml:/etc/radar-gateway/radar-is.yml + - ./etc/gateway/classpath.xml:/usr/local/tomcat/conf/Catalina/localhost/radar-gateway.xml + healthcheck: + # should give an unauthenticated response, rather than a 404 + test: ["CMD-SHELL", "wget --spider localhost:8080/radar-gateway 2>&1 | grep -q 401 || exit 1"] + interval: 1m30s + timeout: 5s + retries: 3 + + #---------------------------------------------------------------------------# + # Catalog server from radar-schemas # + #---------------------------------------------------------------------------# + catalog-server: + build: kafka-radarinit + image: radarcns/kafka-init + networks: + - management + command: radar-schemas-tools serve /schema/merged + volumes: + - ./etc/schema:/schema/conf + environment: + KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181 + KAFKA_SCHEMA_REGISTRY: http://schema-registry-1:8081 + KAFKA_NUM_BROKERS: 3 + RADAR_NUM_PARTITIONS: 3 + RADAR_NUM_REPLICATION_FACTOR: 3 + healthcheck: + test: ["CMD", "curl", "-f", "localhost:9010/source-types"] + interval: 1m30s + timeout: 5s + retries: 3 diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-kacc.sh b/dcompose-stack/radar-cp-hadoop-stack/docker-kacc.sh new file mode 100755 index 000000000..3e0262921 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-kacc.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# kafka-avro-console-consumer inside dockerized radar platform + +if [[ $# -lt 1 || $1 = "-h" || $1 = "--help" || $1 == "--"* ]]; then + printf "Usage: $0 [passthrough options]\n" + printf "\nkafka-avro-console-consumer passthrough options:\n" + docker-compose exec schema-registry-1 kafka-avro-console-consumer + exit 1 +fi + +#Save command line arguments so functions can access it +#To access command line arguments use syntax ${args[0]} etc +args=("$@") + +# get list of available topics +LS_CMD="docker-compose exec kafka-1 kafka-topics --zookeeper zookeeper-1:2181 --list" +topics=($($LS_CMD)) +#printf "%s\n" "${topics[@]}" + +# consumer command to run +KACC_CMD="kafka-avro-console-consumer --zookeeper zookeeper-1:2181 --property schema.registry.url=http://schema-registry-1:8081 --property print.key=true --topic ${args[0]} ${args[@]:1}" +DOCKER_CMD="docker-compose exec schema-registry-1" + +# check if is valid topic +array_contains () { + local array="$1[@]" + local seeking=$2 + local in=1 + for element in "${!array}"; do + element_s=$(echo $element | tr -d '\r') + seeking_s=$(echo $seeking | tr -d '\r') + if [[ $element_s == $seeking_s ]]; then + in=0 + break + fi + done + return $in +} +if ! array_contains topics ${args[0]}; then + echo -e "Topic ${args[0]} not available. Topics on server are:\n" + printf "%s\n" "${topics[@]}" + exit 1 +fi + +# run consumer +echo $DOCKER_CMD $KACC_CMD +exec $DOCKER_CMD $KACC_CMD + + diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-prune.sh b/dcompose-stack/radar-cp-hadoop-stack/docker-prune.sh new file mode 100755 index 000000000..2d1a33172 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-prune.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +. ./util.sh +. ./.env + +check_command_exists docker + +sudo-linux docker system prune --filter "label!=certs" "$@" && \ + sudo-linux rm -rf "$HDFS_DATA_DIR_1" && \ + sudo-linux rm -rf "$HDFS_DATA_DIR_2" && \ + sudo-linux rm -rf "$HDFS_NAME_DIR_1" && \ + sudo-linux rm -rf "$HDFS_NAME_DIR_2" && \ + sudo-linux rm -rf "$MONGODB_DIR" && \ + sudo-linux rm -rf "$MP_POSTGRES_DIR" + diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore index 9b6b65b55..394404aff 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore @@ -1,7 +1,14 @@ /sink-mongo.properties /sink-hdfs.properties -/nginx.conf +/webserver/nginx.conf /radar.yml /smtp.env /rest-api/device-catalog.yml -/rest-api/radar.yml \ No newline at end of file +/rest-api/radar.yml +/rest-api/mp_info.yml +/redcap-integration/radar.yml +/managementportal/config/keystore.jks +/managementportal/config/liquibase/oauth_client_details.csv +/schema/commons/ +/schema/specifications/ + diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index 107117b0c..becd3c8f4 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -1,9 +1,10 @@ SERVER_NAME=localhost +MANAGEMENTPORTAL_KEY_DNAME=CN=localhost,OU=MyName,O=MyOrg,L=MyCity,S=MyState,C=MyCountryCode +MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET= SELF_SIGNED_CERT=yes MAINTAINER_EMAIL=me@example.com -RADAR_TOPIC_LIST=android_empatica_e4_acceleration,android_empatica_e4_acceleration_output,android_empatica_e4_battery_level,android_empatica_e4_battery_level_output,android_empatica_e4_blood_volume_pulse,android_empatica_e4_blood_volume_pulse_output,android_empatica_e4_electrodermal_activity,android_empatica_e4_electrodermal_activity_output,android_empatica_e4_heartrate,android_empatica_e4_inter_beat_interval,android_empatica_e4_inter_beat_interval_output,android_empatica_e4_sensor_status,android_empatica_e4_sensor_status_output,android_empatica_e4_temperature,android_empatica_e4_temperature_output,application_server_status,application_record_counts,application_uptime,application_external_time,android_phone_battery_level,android_phone_acceleration,android_phone_light,android_pebble2_acceleration,android_pebble2_battery_level,android_pebble2_heart_rate,android_pebble2_heart_rate_filtered,active_questionnaire_phq8,schemaless-key,schemaless-value,thincit_code_breaker,thincit_spotter,thincit_symbol_check,thincit_trails -RADAR_RAW_TOPIC_LIST=android_empatica_e4_acceleration,android_empatica_e4_battery_level,android_empatica_e4_blood_volume_pulse,android_empatica_e4_electrodermal_activity,android_empatica_e4_inter_beat_interval,android_empatica_e4_sensor_status,android_empatica_e4_temperature,application_server_status,application_record_counts,application_uptime,application_external_time,android_phone_battery_level,android_phone_acceleration,android_phone_light,android_pebble2_acceleration,android_pebble2_battery_level,android_pebble2_heart_rate,android_pebble2_heart_rate_filtered,active_questionnaire_phq8 -RADAR_AGG_TOPIC_LIST=android_empatica_e4_acceleration_output,android_empatica_e4_battery_level_output,android_empatica_e4_blood_volume_pulse_output,android_empatica_e4_electrodermal_activity_output,android_empatica_e4_heartrate,android_empatica_e4_inter_beat_interval_output,android_empatica_e4_sensor_status_output,android_empatica_e4_temperature_output,application_server_status,application_record_counts,application_uptime,application_external_time,active_questionnaire_phq8 +RADAR_RAW_TOPIC_LIST= +RADAR_AGG_TOPIC_LIST= HOTSTORAGE_USERNAME= HOTSTORAGE_PASSWORD=XXXXXXXX HOTSTORAGE_NAME= @@ -15,4 +16,8 @@ MONGODB_DIR=/usr/local/var/lib/docker/mongodb MP_POSTGRES_DIR=/usr/local/var/lib/docker/postgres POSTGRES_USER= POSTGRES_PASSWORD= -FROM_EMAIL= \ No newline at end of file +FROM_EMAIL= +MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT=false +HEALTHCHECK_HIPCHAT_NOTIFY=no +HEALTHCHECK_HIPCHAT_ROOM_ID="" +HEALTHCHECK_HIPCHAT_TOKEN="" diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/classpath.xml b/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/classpath.xml new file mode 100644 index 000000000..408a20479 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/classpath.xml @@ -0,0 +1,10 @@ + + + + + + + diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/radar-is.yml b/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/radar-is.yml new file mode 100644 index 000000000..a6f64f516 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/radar-is.yml @@ -0,0 +1,2 @@ +resourceName: res_ManagementPortal +publicKeyEndpoint: http://managementportal-app:8080/managementportal/oauth/token_key diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/changelogs/config/liquibase/oauth_client_details.csv.template b/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/changelogs/config/liquibase/oauth_client_details.csv.template deleted file mode 100644 index 17801e241..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/changelogs/config/liquibase/oauth_client_details.csv.template +++ /dev/null @@ -1,6 +0,0 @@ -client_id;resource_ids;client_secret;scope;authorized_grant_types;web_server_redirect_uri;authorities;access_token_validity;refresh_token_validity;additional_information;autoapprove -ManagementPortalapp;res_ManagementPortal;my-secret-token-to-change-in-production;read,write;password,refresh_token,authorization_code,implicit;;ROLE_PROJECT_ADMIN,ROLE_USER,ROLE_SYS_ADMIN;1800;3600;{};true -pRMT;res_ManagementPortal;;read,write;refresh_token,authorization_code;http://localhost:8080;ROLE_USER;43200;5184000;{};true -radar_restapi;res_ManagementPortal,res_gateway;;read;client_credentials;;ROLE_USER;1800;3600;{};true -THINC-IT;res_gateway,res_ManagementPortal;;read,write;password,refresh_token,authorization_code,implicit;;ROLE_PROJECT_ADMIN,ROLE_USER,ROLE_SYS_ADMIN;43200;5184000;{};true -radar_redcap_integrator;res_ManagementPortal;;read,write;client_credentials;;ROLE_EXTERNAL_ERF_INTEGRATOR;1800;2000;{};true diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/config/oauth_client_details.csv.template b/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/config/oauth_client_details.csv.template new file mode 100644 index 000000000..b5f4b8587 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/config/oauth_client_details.csv.template @@ -0,0 +1,7 @@ +client_id;resource_ids;client_secret;scope;authorized_grant_types;redirect_uri;authorities;access_token_validity;refresh_token_validity;additional_information;autoapprove +pRMT;res_ManagementPortal,res_gateway;secret;MEASUREMENT.CREATE,SUBJECT.UPDATE,SUBJECT.READ,PROJECT.READ,SOURCETYPE.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ,USER.READ,ROLE.READ;refresh_token,authorization_code;;;43200;7948800;{"dynamic_registration": true}; +aRMT;res_ManagementPortal,res_gateway;secret;MEASUREMENT.CREATE,SUBJECT.UPDATE,SUBJECT.READ,PROJECT.READ,SOURCETYPE.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ,USER.READ,ROLE.READ;refresh_token,authorization_code;;;43200;7948800;{"dynamic_registration": true}; +THINC-IT;res_ManagementPortal,res_gateway;secret;MEASUREMENT.CREATE,SUBJECT.UPDATE,SUBJECT.READ,PROJECT.READ,SOURCETYPE.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ,USER.READ,ROLE.READ;refresh_token,authorization_code;;;43200;7948800;{"dynamic_registration": true}; +radar_restapi;res_ManagementPortal;secret;SUBJECT.READ,PROJECT.READ,SOURCE.READ,SOURCETYPE.READ;client_credentials;;;43200;259200;{}; +radar_redcap_integrator;res_ManagementPortal;secret;PROJECT.READ,SUBJECT.CREATE,SUBJECT.READ,SUBJECT.UPDATE;client_credentials;;;43200;259200;{}; +radar_dashboard;res_ManagementPortal,res_restApi;secret;SUBJECT.READ,PROJECT.READ,SOURCE.READ,SOURCETYPE.READ;client_credentials;;;43200;259200;{}; diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/radar.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/radar.yml.template index e072c8975..9bda09a38 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/radar.yml.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/radar.yml.template @@ -2,7 +2,6 @@ version: 1.0 released: 2016-11-27 #============================= Application =============================# -log_path: #Possible value are standalone or high_performance mode: standalone @@ -19,8 +18,9 @@ broker: port: 9092 #Kafka internal parameters -auto_commit_interval_ms: 1000 -session_timeout_ms: 10000 +stream_properties: + auto_commit_interval_ms: 1000 + session_timeout_ms: 10000 #============================ Kafka Streams ============================# #The number of threads that a stream must be run according is priority @@ -36,13 +36,18 @@ schema_registry: port: 8081 protocol: http -#========================= Battery level monitor=========================# +rest_proxy: + host: rest-proxy-1 + port: 8082 + protocol: http + +#======================== Battery level monitor ========================# battery_monitor: - level: CRITICAL + level: LOW email_address: - - notifiersemail + - notifier@email - notifier2@email - email_host: smtp + email_host: localhost email_port: 25 email_user: user@example.com topics: @@ -53,9 +58,17 @@ disconnect_monitor: email_address: - notifier@email - notifier2@email - email_host: smtp + email_host: localhost email_port: 25 email_user: user@example.com topics: - - android_empatica_e4_battery_level - timeout: 300000 + - android_empatica_e4_temperature + timeout: 1800 # seconds after which a stream is set disconnected + alert_repetitions: 2 # number of additional emails to send after the first + +#========================= Kafka stream ================================# +stream_masters: + - org.radarcns.stream.empatica.E4StreamMaster + - org.radarcns.stream.phone.PhoneStreamMaster + +persistence_path: /var/lib/radar/data diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/redcap-integration/radar.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/redcap-integration/radar.yml.template index 78e083ffe..7c4add2b7 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/redcap-integration/radar.yml.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/redcap-integration/radar.yml.template @@ -17,5 +17,6 @@ projects: enrolment_event: #Unique identifier for the enrolment event integration_form: #Name of integration REDCap form token: #REDCap API Token used to identify the REDCap user against the REDCap instance - mp_info: - project_id: #Management Portal project identifier + - mp_info: + # Management Portal project identifier + project_name: # diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/mp_info.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/mp_info.yml.template new file mode 100644 index 000000000..ed71a0e01 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/mp_info.yml.template @@ -0,0 +1,10 @@ +version: 0.1-alpha +released: 2017-08-29 + +# ManagementPortal configuration +oauth_client_id: #OAuth2 clientId used by the webApp for making requests +oauth_client_secret: #OAuth2 client secret +management_portal_url: #URL pointing Management Portal +token_endpoint: #URL managing tokens +project_endpoint: #URL managing project function +subject_endpoint: #URL managing subject functions diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/schema/commons/.gitkeep b/dcompose-stack/radar-cp-hadoop-stack/etc/schema/commons/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/schema/specifications/.gitkeep b/dcompose-stack/radar-cp-hadoop-stack/etc/schema/specifications/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs.properties.template b/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs.properties.template index 5feb5bd1f..12052219e 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs.properties.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs.properties.template @@ -2,7 +2,7 @@ name=radar-hdfs-sink-android-15000 connector.class=io.confluent.connect.hdfs.HdfsSinkConnector tasks.max=4 topics=android_empatica_e4_electrodermal_activity,android_empatica_e4_blood_volume_pulse,android_empatica_e4_temperature -flush.size=150 +flush.size=80000 rotate.interval.ms=900000 hdfs.url=hdfs://hdfs-namenode:8020 format.class=org.radarcns.sink.hdfs.AvroFormatRadar diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/cors.conf b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/cors.conf new file mode 100644 index 000000000..6abf87141 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/cors.conf @@ -0,0 +1,53 @@ +# From: https://gist.github.com/Stanback/7145487 +# +# CORS header support +# +# One way to use this is by placing it into a file called "cors_support" +# under your Nginx configuration directory and placing the following +# statement inside your **location** block(s): +# +# include cors_support; +# +# As of Nginx 1.7.5, add_header supports an "always" parameter which +# allows CORS to work if the backend returns 4xx or 5xx status code. +# +# For more information on CORS, please see: http://enable-cors.org/ +# Forked from this Gist: https://gist.github.com/michiel/1064640 +# + +set $cors_method ''; + +if ($request_method = 'GET') { + set $cors_method 'noopt'; +} +if ($request_method = 'POST') { + set $cors_method 'noopt'; +} +if ($request_method = 'HEAD') { + set $cors_method 'noopt'; +} +if ($request_method = 'PUT') { + set $cors_method 'noopt'; +} +if ($request_method = 'DELETE') { + set $cors_method 'noopt'; +} +if ($request_method = 'OPTIONS') { + # Tell client that this pre-flight info is valid for 20 days + add_header 'Access-Control-Allow-Origin' "$http_origin" always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With' always; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; +} +if ($cors_method = 'noopt') { + add_header 'Access-Control-Allow-Origin' "$http_origin" always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With' always; + # required to be able to read Authorization header in frontend + #add_header 'Access-Control-Expose-Headers' 'Authorization' always; +} diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/nginx.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template similarity index 82% rename from dcompose-stack/radar-cp-hadoop-stack/etc/nginx.conf.template rename to dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template index 1ff380f0d..05e494711 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/nginx.conf.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template @@ -5,8 +5,6 @@ events { } http { - index index.html index.htm index.php; - default_type application/octet-stream; log_format main '$remote_addr - $remote_user [$time_local] $status ' '"$request" $body_bytes_sent "$http_referer" ' @@ -60,12 +58,25 @@ http { error_log /var/log/nginx/error.log; location /kafka/ { - proxy_pass http://rest-proxy-1:8082/; + include cors.conf; + proxy_pass http://gateway:8080/radar-gateway/; proxy_set_header Host $host; proxy_http_version 1.1; proxy_set_header Connection ""; } + location ^~ /kafka/consumers { + deny all; + } + location ^~ /kafka/brokers { + deny all; + } + location ~* /kafka/topics/.+/partitions { + deny all; + } location /schema/ { + limit_except GET { + deny all; + } proxy_pass http://schema-registry-1:8081/; proxy_set_header Host $host; proxy_http_version 1.1; @@ -87,6 +98,7 @@ http { proxy_http_version 1.1; } location /api/ { + include cors.conf; proxy_pass http://rest-api:8080/radar/api/; proxy_set_header Host $host; } @@ -98,10 +110,14 @@ http { proxy_pass http://rest-api:8080/radar/frontend/; proxy_set_header Host $host; } - - location /managementportal/{ - proxy_pass http://managementportal-app:8080/; - proxy_set_header Host $host; + location /managementportal/ { + include cors.conf; + proxy_pass http://managementportal-app:8080/managementportal/; + proxy_set_header Host $host; + } + location /kafkamanager/{ + proxy_pass http://kafka-manager:9000; + proxy_set_header Host $host; } } } diff --git a/dcompose-stack/radar-cp-hadoop-stack/hdfs_restructure.sh b/dcompose-stack/radar-cp-hadoop-stack/hdfs_restructure.sh index 1e8c198e1..bc3afebd6 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/hdfs_restructure.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/hdfs_restructure.sh @@ -8,12 +8,12 @@ fi . ./util.sh # HDFS restructure version -JAR_VERSION=0.1.1 +JAR_VERSION=0.3.1 # HDFS restructure JAR JAR="restructurehdfs-all-${JAR_VERSION}.jar" if [ ! -e "lib/${JAR}" ]; then - echo "Downloading HDFS restructuring JAR" + echo "Downloading HDFS restructuring JAR version ${JAR_VERSION}" sudo-linux curl -L -# -o lib/${JAR} "https://github.com/RADAR-CNS/Restructure-HDFS-topic/releases/download/v${JAR_VERSION}/${JAR}" fi diff --git a/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh b/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh index 2ef389f74..3fe872424 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh @@ -1,8 +1,30 @@ #!/bin/bash . ./util.sh + +sudo-linux chmod og-rw ./.env +sudo-linux chmod og-rwx ./etc +if [ -e ./output ]; then + sudo-linux chmod og-rwx ./output +else + sudo-linux mkdir -m 0700 ./output +fi + +# Initialize and check all config files +check_config_present .env etc/env.template +check_config_present etc/radar.yml +check_config_present etc/managementportal/config/oauth_client_details.csv +check_config_present etc/rest-api/mp_info.yml +check_config_present etc/redcap-integration/radar.yml +copy_template_if_absent etc/sink-mongo.properties +copy_template_if_absent etc/sink-hdfs.properties +copy_template_if_absent etc/rest-api/radar.yml +copy_template_if_absent etc/rest-api/device-catalog.yml +copy_template_if_absent etc/webserver/nginx.conf + . ./.env +# Check provided directories and configurations check_parent_exists HDFS_DATA_DIR_1 ${HDFS_DATA_DIR_1} check_parent_exists HDFS_DATA_DIR_2 ${HDFS_DATA_DIR_2} check_parent_exists HDFS_NAME_DIR_1 ${HDFS_NAME_DIR_1} @@ -15,51 +37,85 @@ if [ -z ${SERVER_NAME} ]; then exit 1 fi -if [ -z $(sudo-linux docker network ls --format '{{.Name}}' | grep "^hadoop$") ]; then +# Create networks and volumes +if ! sudo-linux docker network ls --format '{{.Name}}' | grep -q "^hadoop$"; then echo "==> Creating docker network - hadoop" sudo-linux docker network create hadoop > /dev/null else echo "==> Creating docker network - hadoop ALREADY EXISTS" fi -echo "==> Configuring MongoDB Connector" +echo "==> Checking docker external volumes" +if ! sudo-linux docker volume ls -q | grep -q "^certs$"; then + sudo-linux docker volume create --name=certs --label certs +fi +if ! sudo-linux docker volume ls -q | grep -q "^certs-data$"; then + sudo-linux docker volume create --name=certs-data --label certs +fi +# Initializing Kafka +echo "==> Setting up topics" +sudo-linux docker-compose run --rm kafka-init + +echo "==> Configuring MongoDB Connector" # Update sink-mongo.properties -copy_template_if_absent etc/sink-mongo.properties inline_variable 'mongo.username=' $HOTSTORAGE_USERNAME etc/sink-mongo.properties inline_variable 'mongo.password=' $HOTSTORAGE_PASSWORD etc/sink-mongo.properties inline_variable 'mongo.database=' $HOTSTORAGE_NAME etc/sink-mongo.properties # Set topics -inline_variable 'topics=' "${RADAR_AGG_TOPIC_LIST}" etc/sink-mongo.properties +if [ -z "${COMBINED_AGG_TOPIC_LIST}"]; then + COMBINED_AGG_TOPIC_LIST=$(sudo-linux docker run --rm radarcns/kafka-init list_aggregated.sh 2>/dev/null) + if [ -n "${RADAR_AGG_TOPIC_LIST}" ]; then + COMBINED_AGG_TOPIC_LIST="${RADAR_AGG_TOPIC_LIST},${COMBINED_AGG_TOPIC_LIST}" + fi +fi +inline_variable 'topics=' "${COMBINED_AGG_TOPIC_LIST}" etc/sink-mongo.properties echo "==> Configuring HDFS Connector" -copy_template_if_absent etc/sink-hdfs.properties -inline_variable 'topics=' "${RADAR_RAW_TOPIC_LIST}" etc/sink-hdfs.properties +if [ -z "${COMBINED_RAW_TOPIC_LIST}"]; then + COMBINED_RAW_TOPIC_LIST=$(sudo-linux docker run --rm radarcns/kafka-init list_raw.sh 2>/dev/null) + if [ -n "${RADAR_RAW_TOPIC_LIST}" ]; then + COMBINED_RAW_TOPIC_LIST="${RADAR_RAW_TOPIC_LIST},${COMBINED_RAW_TOPIC_LIST}" + fi +fi +inline_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/sink-hdfs.properties -echo "==> Configuring REST-API" -copy_template_if_absent etc/rest-api/radar.yml -copy_template_if_absent etc/rest-api/device-catalog.yml +echo "==> Configuring Management Portal" -echo "==> Configuring REDCap-Integration" -copy_template_if_absent etc/redcap-integration/radar.yml + +keystorefile=etc/managementportal/config/keystore.jks +if [ -f "$keystorefile" ]; then + echo "--> Keystore for signing JWTs already exists. Not creating a new one." +else + echo "--> Generating keystore to hold RSA keypair for JWT signing" + if [ -n "${MANAGEMENTPORTAL_KEY_DNAME}" ]; then + sudo-linux keytool -genkeypair -dname "${MANAGEMENTPORTAL_KEY_DNAME}" -alias selfsigned -keyalg RSA -keystore "$keystorefile" -keysize 4096 -storepass radarbase -keypass radarbase + else + sudo-linux keytool -genkeypair -alias selfsigned -keyalg RSA -keystore "$keystorefile" -keysize 4096 -storepass radarbase -keypass radarbase + fi + sudo-linux chmod 400 "${keystorefile}" +fi + +echo "==> Configuring REST-API" # Set MongoDb credential -inline_variable 'usr:[[:space:]]' $HOTSTORAGE_USERNAME etc/rest-api/radar.yml -inline_variable 'pwd:[[:space:]]' $HOTSTORAGE_PASSWORD etc/rest-api/radar.yml -inline_variable 'db:[[:space:]]' $HOTSTORAGE_NAME etc/rest-api/radar.yml +inline_variable 'usr:[[:space:]]' "$HOTSTORAGE_USERNAME" etc/rest-api/radar.yml +inline_variable 'pwd:[[:space:]]' "$HOTSTORAGE_PASSWORD" etc/rest-api/radar.yml +inline_variable 'db:[[:space:]]' "$HOTSTORAGE_NAME" etc/rest-api/radar.yml # Set variable for Swagger inline_variable 'host:[[:space:]]*' "${SERVER_NAME}" etc/rest-api/radar.yml +echo "==> Configuring REDCap-Integration" + echo "==> Configuring nginx" -copy_template_if_absent etc/nginx.conf -inline_variable 'server_name[[:space:]]*' "${SERVER_NAME};" etc/nginx.conf -sed_i 's|\(/etc/letsencrypt/live/\)[^/]*\(/.*\.pem\)|\1'"${SERVER_NAME}"'\2|' etc/nginx.conf +inline_variable 'server_name[[:space:]]*' "${SERVER_NAME};" etc/webserver/nginx.conf +sed_i 's|\(/etc/letsencrypt/live/\)[^/]*\(/.*\.pem\)|\1'"${SERVER_NAME}"'\2|' etc/webserver/nginx.conf init_certificate "${SERVER_NAME}" echo "==> Starting RADAR-CNS Platform" -sudo-linux docker-compose up --force-recreate -d "$@" +sudo-linux docker-compose up -d "$@" request_certificate "${SERVER_NAME}" "${SELF_SIGNED_CERT:-yes}" echo "### SUCCESS ###" diff --git a/dcompose-stack/radar-cp-hadoop-stack/install-systemd-wrappers.sh b/dcompose-stack/radar-cp-hadoop-stack/install-systemd-wrappers.sh new file mode 100755 index 000000000..c443bb459 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/install-systemd-wrappers.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +. ./util.sh + +check_command_exists systemctl + +copy_template_if_absent /etc/systemd/system/radar-docker.service lib/systemd/radar-docker.service.template +copy_template_if_absent /etc/systemd/system/radar-output.service lib/systemd/radar-output.service.template +copy_template_if_absent /etc/systemd/system/radar-output.timer lib/systemd/radar-output.timer.template +copy_template_if_absent /etc/systemd/system/radar-check-health.service lib/systemd/radar-check-health.service.template +copy_template_if_absent /etc/systemd/system/radar-check-health.timer lib/systemd/radar-check-health.timer.template +copy_template_if_absent /etc/systemd/system/radar-renew-certificate.service lib/systemd/radar-renew-certificate.service.template +copy_template_if_absent /etc/systemd/system/radar-renew-certificate.timer lib/systemd/radar-renew-certificate.timer.template + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-docker.service +inline_variable 'ExecStart=' "$DIR/lib/systemd/start-radar-stack.sh" /etc/systemd/system/radar-docker.service + +inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-output.service +inline_variable 'ExecStart=' "$DIR/hdfs_restructure.sh /topicAndroidNew output" /etc/systemd/system/radar-output.service + +inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-check-health.service +inline_variable 'ExecStart=' "$DIR/check-health.sh" /etc/systemd/system/radar-check-health.service + +inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-renew-certificate.service +inline_variable 'ExecStart=' "$DIR/renew_ssl_certificate.sh" /etc/systemd/system/radar-renew-certificate.service + + +sudo systemctl daemon-reload +sudo systemctl enable radar-docker +sudo systemctl enable radar-output.timer +sudo systemctl enable radar-check-health.timer +sudo systemctl enable radar-renew-certificate.timer +sudo systemctl start radar-docker +sudo systemctl start radar-output.timer +sudo systemctl start radar-check-health.timer +sudo systemctl start radar-renew-certificate.timer diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile index de2e65d87..7112136d1 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile +++ b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile @@ -1,7 +1,44 @@ -FROM confluentinc/cp-kafka:3.1.2-1 +FROM openjdk:8-alpine + +MAINTAINER Joris Borgdorff + +ENV KAFKA_SCHEMA_REGISTRY=http://schema-registry-1:8081 +ENV KAFKA_NUM_PARTITIONS=3 +ENV KAFKA_NUM_REPLICATION=2 +ENV KAFKA_NUM_BROKERS=3 +ENV KAFKA_ZOOKEEPER_CONNECT=zookeeper-1:2181 + +RUN apk add --no-cache \ + bash \ + curl \ + rsync \ + tar + +RUN mkdir -p /schema/merged /schema/java/src /schema/java/classes /usr/share/java + +WORKDIR /schema + +ENV JQ_VERSION=1.5 +RUN curl -L#o /usr/bin/jq https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 \ + && chmod +x /usr/bin/jq +RUN curl -#o /usr/share/java/avro-tools.jar \ + "$(curl -s http://www.apache.org/dyn/closer.cgi/avro/\?as_json \ + | jq --raw-output ".preferred")avro/avro-1.8.2/java/avro-tools-1.8.2.jar" + +ENV RADAR_SCHEMAS_VERSION=0.2.3 +RUN curl -#L https://github.com/RADAR-CNS/RADAR-Schemas/releases/download/v${RADAR_SCHEMAS_VERSION}/radar-schemas-tools-${RADAR_SCHEMAS_VERSION}.tar.gz \ + | tar xz -C /usr --strip-components 1 + +RUN mkdir original \ + && curl -#L https://github.com/RADAR-CNS/RADAR-Schemas/archive/v${RADAR_SCHEMAS_VERSION}.tar.gz \ + | tar xz -C original --strip-components 1 + +VOLUME /schema/conf # Copy bash file -COPY ./topic_init.sh / -RUN chmod +x /topic_init.sh +COPY ./topic_init.sh ./init.sh ./list_aggregated.sh ./list_raw.sh /usr/bin/ +RUN chmod +x /usr/bin/*.sh + +ENTRYPOINT ["init.sh"] +CMD ["topic_init.sh"] -CMD ["/topic_init.sh", "shutdown -h now"] diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh new file mode 100755 index 000000000..c5e0b5213 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +rsync -a /schema/original/commons /schema/original/specifications /schema/merged +rsync -a /schema/conf/ /schema/merged + +# Compiling updated schemas +echo "Compiling schemas..." >&2 +# Separate enums so that they can be referenced in later files +read -r -a enums <<< $(find merged/commons -name "*.avsc" -exec grep -q '^ "type": "enum"' "{}" \; -print | tr '\n' ' ') +read -r -a notenums <<< $(find merged/commons -name "*.avsc" -exec grep -qv '^ "type": "enum"' "{}" \; -print | tr '\n' ' ') +java -jar /usr/share/java/avro-tools.jar compile -string schema ${enums[@]} ${notenums[@]} java/src 2>/dev/null +find java/src -name "*.java" -print0 | xargs -0 javac -cp /usr/lib/*:java/classes -d java/classes -sourcepath java/src +# Update the radar schemas so the tools find the new classes in classpath +jar uf /usr/lib/radar-schemas-commons-${RADAR_SCHEMAS_VERSION}.jar -C java/classes . + +exec "$@" diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/list_aggregated.sh b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/list_aggregated.sh new file mode 100644 index 000000000..541db7bab --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/list_aggregated.sh @@ -0,0 +1,22 @@ +#/bin/bash + +SAVEIFS=$IFS +# Change IFS to new line. +raw_str=$(radar-schemas-tools list -q --raw merged) +all_str=$(radar-schemas-tools list -q merged) +IFS=$'\n' +raw=($raw_str) +all=($all_str) +# Restore IFS +IFS=$SAVEIFS + +aggregated=() +for i in "${all[@]}"; do + skip= + for j in "${raw[@]}"; do + [[ $i == $j ]] && { skip=1; break; } + done + [[ -n $skip ]] || aggregated+=("$i") +done +IFS=',' +echo "${aggregated[*]}" \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/list_raw.sh b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/list_raw.sh new file mode 100644 index 000000000..b8867d27c --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/list_raw.sh @@ -0,0 +1,2 @@ +#/bin/bash +radar-schemas-tools list -q --raw merged | tr '\n' ',' diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/topic_init.sh b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/topic_init.sh index 965769552..8b6b12e34 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/topic_init.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/topic_init.sh @@ -1,68 +1,44 @@ #!/bin/bash -# Check if first execution -if [ -f /.radar_topic_set ]; then - echo "*********************************************" - echo "** RADAR-CNS topics are ready to be used **" - echo "*********************************************" - exit 0 -fi +# Create topics +echo "Creating RADAR-CNS topics..." -# Wait until all brokers are up & running -if [ -z ${KAFKA_ZOOKEEPER_CONNECT} ]; then - echo "KAFKA_ZOOKEEPER_CONNECT is not defined" - exit 2 +if ! radar-schemas-tools create -p $KAFKA_NUM_PARTITIONS -r $KAFKA_NUM_REPLICATION -b $KAFKA_NUM_BROKERS "${KAFKA_ZOOKEEPER_CONNECT}" merged; then + echo "FAILED TO CREATE TOPICS" + exit 1 fi -interval=1 -while [ "$LENGTH" != "$KAFKA_BROKERS" ]; do - ZOOKEEPER_CHECK=$(zookeeper-shell $KAFKA_ZOOKEEPER_CONNECT <<< "ls /brokers/ids") - ZOOKEEPER_CHECK="${ZOOKEEPER_CHECK##*$'\n'}" - ZOOKEEPER_CHECK="$(echo -e "${ZOOKEEPER_CHECK}" | tr -d '[:space:]' | tr -d '[' | tr -d ']')" - - IFS=',' read -r -a array <<< $ZOOKEEPER_CHECK - LENGTH=${#array[@]} - if [ "$LENGTH" != "$KAFKA_BROKERS" ]; then - echo "Expected $KAFKA_BROKERS brokers but found only $LENGTH. Waiting $interval second before retrying ..." - sleep $interval - if (( interval < 30 )); then - ((interval=interval*2)) - fi - fi +echo "Topics created." + +echo "Registering RADAR-CNS schemas..." + +tries=10 +timeout=1 +max_timeout=32 +while true; do + if curl -Ifs "${KAFKA_SCHEMA_REGISTRY}" > /dev/null; then + break; + fi + tries=$((tries - 1)) + if [ $tries = 0 ]; then + echo "FAILED TO REACH SCHEMA REGISTRY. SCHEMAS NOT REGISTERED" + exit 1 + fi + echo "Failed to reach schema registry. Retrying in ${timeout} seconds." + sleep $timeout + if [ $timeout -lt $max_timeout ]; then + timeout=$((timeout * 2)) + fi done - -# Check if variables exist -if [ -z ${RADAR_TOPICS} ]; then - echo "RADAR_TOPICS is not defined" - exit 2 -fi - -if [ -z ${RADAR_PARTITIONS} ]; then - echo "RADAR_PARTITIONS is not defined" - exit 2 +if ! radar-schemas-tools register "${KAFKA_SCHEMA_REGISTRY}" merged; then + echo "FAILED TO REGISTER SCHEMAS" + exit 1 fi -if [ -z ${RADAR_REPLICATION_FACTOR} ]; then - echo "RADAR_REPLICATION_FACTOR is not defined" - exit 2 -fi - -# Create topics -echo "Creating RADAR-CNS topics..." -IFS=',' read -r -a array <<< "$RADAR_TOPICS" - -for element in "${array[@]}" -do - echo "===> Creating $element" - kafka-topics --zookeeper $KAFKA_ZOOKEEPER_CONNECT --create --topic $element --partitions $RADAR_PARTITIONS --replication-factor $RADAR_REPLICATION_FACTOR --if-not-exists -done - -touch /.radar_topic_set - -echo "Topics created!" +echo "Schemas registered." echo "*******************************************" -echo "** RADAR-CNS topics have been created **" +echo "** RADAR-CNS topics and schemas ready **" echo "*******************************************" diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.service.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.service.template new file mode 100644 index 000000000..e26609aad --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.service.template @@ -0,0 +1,12 @@ +[Unit] +Description=RADAR-Docker service +After=radar-docker.service +Requires=docker.service + +[Service] + +WorkingDirectory= + +ExecStart= + +NotifyAccess=all diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.timer.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.timer.template new file mode 100644 index 000000000..be08892cc --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.timer.template @@ -0,0 +1,11 @@ +[Unit] +Description=RADAR-Docker health checking every 5 minutes +BindsTo=radar-docker.service + +[Timer] +OnBootSec=10min +OnUnitActiveSec=5min +Unit=radar-check-health.service + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-docker.service.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-docker.service.template new file mode 100644 index 000000000..ad02b25d1 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-docker.service.template @@ -0,0 +1,23 @@ +[Unit] +Description=RADAR-Docker service +BindsTo=docker.service +After=docker.service + +[Service] +TimeoutStartSec=0 +TimeoutStopSec=90 +Restart=always +RestartSec=10 + +WorkingDirectory= + +ExecStart=./lib/systemd/start-radar-stack.sh + +ExecReload=/usr/local/bin/docker-compose restart + +ExecStop=/usr/local/bin/docker-compose down + +NotifyAccess=all + +[Install] +WantedBy=multi-user.target diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template new file mode 100644 index 000000000..d6f68b75c --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template @@ -0,0 +1,13 @@ +[Unit] +Description=RADAR-Docker service +After=radar-docker.service +Requires=docker.service + +[Service] + +WorkingDirectory= + +ExecStart= + +NotifyAccess=all + diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.timer.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.timer.template new file mode 100644 index 000000000..3ef981a94 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.timer.template @@ -0,0 +1,11 @@ +[Unit] +Description=RADAR-Docker hourly data restructuring +BindsTo=radar-docker.service + +[Timer] +OnBootSec=15min +OnUnitActiveSec=1h +Unit=radar-output.service + +[Install] +WantedBy=multi-user.target diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.service.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.service.template new file mode 100644 index 000000000..e26609aad --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.service.template @@ -0,0 +1,12 @@ +[Unit] +Description=RADAR-Docker service +After=radar-docker.service +Requires=docker.service + +[Service] + +WorkingDirectory= + +ExecStart= + +NotifyAccess=all diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.timer.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.timer.template new file mode 100644 index 000000000..2c3939f1a --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.timer.template @@ -0,0 +1,16 @@ +[Unit] +Description=RADAR-Docker renew ssl certificate daily +BindsTo=radar-docker.service + +[Timer] +OnCalendar=daily +RandomizedDelaySec=12h +Persistent=true +Unit=radar-check-health.service + +[Install] +WantedBy=multi-user.target + + + + diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/start-radar-stack.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/start-radar-stack.sh new file mode 100755 index 000000000..2063ac666 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/start-radar-stack.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "$DIR/../.." +./install-radar-stack.sh +exec docker-compose up \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/util.sh b/dcompose-stack/radar-cp-hadoop-stack/util.sh old mode 100644 new mode 100755 index 36d2c20b7..ec8ad0cd7 --- a/dcompose-stack/radar-cp-hadoop-stack/util.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/util.sh @@ -1,5 +1,3 @@ -#!/bin/bash - # this will trap any errors or commands with non-zero exit status # by calling function catch_errors() trap catch_errors ERR; @@ -15,7 +13,7 @@ check_command_exists() { if sudo-linux /bin/bash -c "command -v "$1" > /dev/null 2>&1"; then echo "$1 version: $(sudo-linux "$1" --version)" else - echo "RADAR-CNS cannot start without $1. Please, install $1 and then try again" + echo "RADAR Platform cannot start without $1. Please, install $1 and then try again" exit 1 fi } @@ -31,6 +29,11 @@ check_parent_exists() { echo "RADAR-CNS stores volumes at ${PARENT}. If this folder does not exist, please create the entire path and then try again" exit 1 fi + if [ -d "$2" ]; then + sudo-linux chmod 700 "$2" + else + sudo-linux mkdir -p -m 0700 "$2" + fi } # sudo if on Linux, not on OS X @@ -67,12 +70,35 @@ inline_variable() { # ".template") to intended configuration file, if the file does not # yet exist. copy_template_if_absent() { - if [ ! -e "$1" ]; then - sudo-linux cp -p "${1}.template" "$1" - elif [ "$1" -ot "${1}.template" ]; then - echo "Configuration file ${1} is older than its template ${1}.template." - echo "Please edit ${1} to ensure it matches the template, remove it or" - echo "run touch on it." + template=${2:-${1}.template} + if [ ! -f "$1" ]; then + if [ -e "$1" ]; then + echo "Configuration file ${1} is not a regular file." + exit 1 + else + sudo-linux cp -p "${template}" "$1" + fi + elif [ "$1" -ot "${template}" ]; then + echo "Configuration file ${1} is older than its template" + echo "${template}. Please edit ${1}" + echo "to ensure it matches the template, remove it or run touch on it." + exit 1 + fi +} + +check_config_present() { + template=${2:-${1}.template} + if [ ! -f "$1" ]; then + if [ -e "$1" ]; then + echo "Configuration file ${1} is not a regular file." + else + echo "Configuration file ${1} is not present." + echo "Please copy it from ${template} and modify it as needed." + fi + exit 1 + elif [ "$1" -ot "${template}" ]; then + echo "Configuration file ${1} is older than its template ${template}." + echo "Please edit ${1} to ensure it matches the template or run touch on it." exit 1 fi } diff --git a/images/kafka-manager/Dockerfile b/images/kafka-manager/Dockerfile new file mode 100644 index 000000000..28f2de9c3 --- /dev/null +++ b/images/kafka-manager/Dockerfile @@ -0,0 +1,28 @@ +FROM hseeberger/scala-sbt as builder +ENV KM_VERSION=1.3.3.14 + +RUN mkdir -p /tmp && \ + cd /tmp && \ + wget https://github.com/yahoo/kafka-manager/archive/${KM_VERSION}.tar.gz && \ + tar xxf ${KM_VERSION}.tar.gz && \ + cd /tmp/kafka-manager-${KM_VERSION} && \ + sbt clean dist && \ + unzip -d / ./target/universal/kafka-manager-${KM_VERSION}.zip && \ + rm -fr /tmp/${KM_VERSION} /tmp/kafka-manager-${KM_VERSION} && \ + mv /kafka-manager-${KM_VERSION} /kafka-manager + +COPY ./conf/application.conf /kafka-manager-${KM_VERSION}/conf/application.conf +WORKDIR /kafka-manager-${KM_VERSION} + +FROM openjdk:8-alpine +MAINTAINER Yatharth Ranjan + +ENV ZK_HOSTS=zookeeper-1:2181 +RUN apk add --no-cache bash + +COPY --from=builder /kafka-manager /kafka-manager +COPY ./conf/application.conf /kafka-manager/conf/application.conf +WORKDIR /kafka-manager + +EXPOSE 9000 +ENTRYPOINT ["./bin/kafka-manager","-Dconfig.file=conf/application.conf"] diff --git a/images/kafka-manager/README.md b/images/kafka-manager/README.md new file mode 100644 index 000000000..cea3f60cf --- /dev/null +++ b/images/kafka-manager/README.md @@ -0,0 +1,41 @@ +# Kafka manager + +The [kafka-manager](https://github.com/yahoo/kafka-manager) is an interactive web based tool for managing Apache Kafka. Kafka manager has beed integrated in the stack. However, the following istructions are included so that its easy for someone in the futuer to update the stack or for using kafka-manager in other projects. +Instructions for deploying kafka-manager in a docker container and proxied through nginx- + + +1. Clone the GitHub repo - `$ git clone https://github.com/yahoo/kafka-manager.git` +2. Change the working directory - `$ cd kafka-manager` +3. Create a zip distribution using scala - `$ ./sbt clean dist` +4. Note the path of the zip file created. +5. Unzip the zip file to the stack location at `RADAR-Docker/dcompose-stack/radar-cp-hadoop-stack/` +6. change directory to unzipped folder (in my case `$ cd kafka-manager-1.3.3.14/`) +7. Create a file named Dockerfile for specifying the docker build - `$ sudo vim Dockerfile` +8. Add the following content to the Dockerfile - +```dockerfile + FROM hseeberger/scala-sbt + + RUN mkdir /kafka-manager-1.3.3.14 + ADD . /kafka-manager-1.3.3.14 + ENV ZK_HOSTS=zookeeper-1:2181 + + WORKDIR /kafka-manager-1.3.3.14 + + EXPOSE 9000 + ENTRYPOINT ["./bin/kafka-manager","-Dconfig.file=conf/application.conf"] + ``` +9. Note- Change the version of `kafka-manger-{Version}` in the Dockerfile above according to the version you cloned and specified by the unzipped folder. +10. Change the `play.http.context` parameter in the conf/application.conf file to point to the location path you are going to specify in the nginx.conf file later. In my case it was - `play.http.context = "/kafkamanager/“` +11. Now edit the `etc/nginx.conf.template` file to include the path to kafka-manager so that it is accessible from the browser. Add the following inside the server tag of nginx.conf file - +```nginx + location /kafkamanager/{ + proxy_pass http://kafka-manager:9000; + proxy_set_header Host $host; + } +``` +12. Now start the stack with `dcompose-stack/radar-cp-hadoop-stack//install-radar-stack.sh`. This will build a docker image for kafka and start it in a container. You can access it with a browser at `https://host/kafkamanager/`. Open the link and add all the information. In this case the zookeeper host is at `zookeeper-1:2181`. This will look something like the image - + +![Add a Cluster](/img/add_cluster.png) + +Note- You can also take the easy route and just pull the docker image from docker hub located at `radarcns/kafka-manager`. But remember that the context path is `/kafka-manager` so you will need to specify this in your `nginx.conf` file + diff --git a/images/kafka-manager/conf/application.conf b/images/kafka-manager/conf/application.conf new file mode 100644 index 000000000..c1eba7a1f --- /dev/null +++ b/images/kafka-manager/conf/application.conf @@ -0,0 +1,43 @@ +# Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0 +# See accompanying LICENSE file. + +# This is the main configuration file for the application. +# ~~~~~ + +# Secret key +# ~~~~~ +# The secret key is used to secure cryptographics functions. +# If you deploy your application to several instances be sure to use the same key! +play.crypto.secret="^ , Joris B , Francesco N @@ -20,10 +20,10 @@ LABEL description="RADAR-CNS Backend streams and monitor" # Install RADAR-Backend RUN echo "==> Installing Components" \ # Download Git RADAR-Backend release - && echo "==> Downloading RADAR-CNS/RADAR-Backend v0.1-beta.1 release from GitHub" \ + && echo "==> Downloading RADAR-CNS/RADAR-Backend 0.1.2 release from GitHub" \ && cd /usr/local && mkdir RADAR-Backend -ADD https://github.com/RADAR-CNS/RADAR-Backend/releases/download/0.1-alpha.2/radar-backend-0.1-alpha.2.jar /usr/share/java/ +ADD https://github.com/RADAR-CNS/RADAR-Backend/releases/download/0.1.2/radar-backend-0.1.2-all.jar /usr/share/java/ # Load topics validator COPY ["./init.sh", "./kafka_status.sh", "/home/"] diff --git a/images/radar-backend-kafka/kafka_status.sh b/images/radar-backend-kafka/kafka_status.sh index 6f24ffc5f..d00d1867b 100755 --- a/images/radar-backend-kafka/kafka_status.sh +++ b/images/radar-backend-kafka/kafka_status.sh @@ -2,43 +2,57 @@ # Check if variables exist if [ -z "$KAFKA_REST_PROXY" ]; then - echo "KAFKA_REST_PROXY is not defined" - exit 2 + echo "KAFKA_REST_PROXY is not defined" + exit 2 fi -if [ -z "$TOPIC_LIST" ]; then - echo "TOPIC_LIST is not defined" - exit 2 +if [ -z "$KAFKA_SCHEMA_REGISTRY" ]; then + echo "KAFKA_SCHEMA_REGISTRY is not defined" + exit 4 fi -# Fetch env topic list -IFS=', ' read -r -a needed <<< $TOPIC_LIST +KAFKA_BROKERS=${KAFKA_BROKERS:-3} -# Fetch env topic list -count=0 -interval=1 -while [ "$count" != "${#needed[@]}" ] ; do +max_timeout=32 - echo "Waiting $interval second before retrying ..." - sleep $interval - if (( interval < 30 )); then - ((interval=interval*2)) +tries=10 +timeout=1 +while true; do + IFS=, read -r -a array <<< $(curl -s "${KAFKA_REST_PROXY}/brokers" | sed 's/^.*\[\(.*\)\]\}/\1/') + LENGTH=${#array[@]} + if [ "$LENGTH" -eq "$KAFKA_BROKERS" ]; then + echo "Kafka brokers available." + break fi + tries=$((tries - 1)) + if [ $tries -eq 0 ]; then + echo "FAILED: KAFKA BROKERs NOT READY." + exit 5 + fi + echo "Kafka brokers or Kafka REST proxy not ready. Retrying in ${timeout} seconds." + sleep $timeout + if [ $timeout -lt $max_timeout ]; then + timeout=$((timeout * 2)) + fi +done - count=0 - TOPICS=$(curl -sSX GET -H "Content-Type: application/json" "$KAFKA_REST_PROXY/topics") - TOPICS="$(echo -e "${TOPICS}" | tr -d '"' | tr -d '[' | tr -d ']' | tr -d '[:space:]' )" - - IFS=',' read -r -a array <<< $TOPICS - for topic in "${array[@]}" - do - for need in "${needed[@]}" - do - if [ "$topic" = "$need" ] ; then - ((count++)) - fi - done - done +tries=10 +timeout=1 +while true; do + if wget --spider -q "${KAFKA_SCHEMA_REGISTRY}/subjects" 2>/dev/null; then + break + fi + tries=$((tries - 1)) + if [ $tries -eq 0 ]; then + echo "FAILED TO REACH SCHEMA REGISTRY. SCHEMAS NOT REGISTERED." + exit 6 + fi + echo "Failed to reach schema registry. Retrying in ${timeout} seconds." + sleep $timeout + if [ $timeout -lt $max_timeout ]; then + timeout=$((timeout * 2)) + fi done -echo "All topics are now available. Ready to go!" \ No newline at end of file + +echo "Kafka is available. Ready to go!" diff --git a/images/radar-hdfs-connector/Dockerfile b/images/radar-hdfs-connector/Dockerfile index 7ffad215d..be72891e4 100644 --- a/images/radar-hdfs-connector/Dockerfile +++ b/images/radar-hdfs-connector/Dockerfile @@ -11,19 +11,14 @@ # limitations under the License. -FROM confluentinc/cp-kafka-connect:3.1.2-1 +FROM confluentinc/cp-kafka-connect:3.2.1 -MAINTAINER Nivethika M , Joris B , Francesco N +MAINTAINER Nivethika M , Joris B LABEL description="RADAR-CNS Backend- HDFS Sink Connector" -# Install RADAR-Backend -RUN echo "==> Installing Components" \ - # Download Git RADAR-Backend release - && echo "==> Downloading RADAR-CNS/RADAR-HDFS-Sink-Connector 0.1-alpha.1 release from GitHub" - # Deploy RADAR-Backend - HDFS Sink Connector -ADD https://github.com/RADAR-CNS/RADAR-HDFS-Sink-Connector/releases/download/0.1-alpha.1/radar-hdfs-sink-connector-0.1-alpha.1-sources.jar /etc/kafka-connect/jars/ +ADD https://github.com/RADAR-CNS/RADAR-HDFS-Sink-Connector/releases/download/v0.1/radar-hdfs-sink-connector-0.1-sources.jar /etc/kafka-connect/jars/ # Load topics validator COPY ./kafka_status.sh /home/kafka_status.sh diff --git a/images/radar-hdfs-connector/kafka_status.sh b/images/radar-hdfs-connector/kafka_status.sh index 645f8e47f..e488b1eef 100755 --- a/images/radar-hdfs-connector/kafka_status.sh +++ b/images/radar-hdfs-connector/kafka_status.sh @@ -2,56 +2,62 @@ # Check if variables exist if [ -z "$CONNECT_ZOOKEEPER_CONNECT" ]; then - echo "CONNECT_ZOOKEEPER_CONNECT is not defined" - exit 2 + echo "CONNECT_ZOOKEEPER_CONNECT is not defined" + exit 2 fi -if [ -z "$TOPIC_LIST" ]; then - echo "TOPIC_LIST is not defined" - exit 2 +if [ -z "$CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL" ]; then + echo "CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL is not defined" + exit 4 fi -# Save current IFS -SAVEIFS=$IFS +KAFKA_BROKERS=${KAFKA_BROKERS:-3} -# Fetch env topic list -IFS=', ' read -r -a needed <<< $TOPIC_LIST +max_timeout=32 -# Fetch env topic list -IFS=$'\n' -count=0 -interval=1 -max_retryes=10 -while [ "$count" != "${#needed[@]}" ] ; do +tries=10 +timeout=1 +while true; do + ZOOKEEPER_CHECK=$(zookeeper-shell $CONNECT_ZOOKEEPER_CONNECT <<< "ls /brokers/ids") + ZOOKEEPER_CHECK="${ZOOKEEPER_CHECK##*$'\n'}" + ZOOKEEPER_CHECK="$(echo -e "${ZOOKEEPER_CHECK}" | tr -d '[:space:]' | tr -d '[' | tr -d ']')" - if [ "$max_retryes" -eq "0" ] ; then - IFS=$SAVEIFS - echo "Force rebooting ... " - exit 2 + IFS=',' read -r -a array <<< $ZOOKEEPER_CHECK + LENGTH=${#array[@]} + if [ "$LENGTH" -eq "$KAFKA_BROKERS" ]; then + echo "Kafka brokers available." + break fi - count=0 - topics=$(kafka-topics --list --zookeeper $CONNECT_ZOOKEEPER_CONNECT) - topics=($topics) - - for topic in "${topics[@]}" - do - for need in "${needed[@]}" - do - if [ "$topic" = "$need" ] ; then - ((count++)) - fi - done - done - - if [ "$count" != "${#needed[@]}" ] ; then - echo "Waiting $interval second before retrying ..." - sleep $interval - if (( interval < 30 )); then - ((interval=interval*2)) - fi - ((max_retryes--)) + tries=$((tries - 1)) + if [ $tries -eq 0 ]; then + echo "FAILED: KAFKA BROKERs NOT READY." + exit 5 + fi + echo "Expected $KAFKA_BROKERS brokers but found only $LENGTH. Waiting $timeout second before retrying ..." + sleep $timeout + if [ $timeout -lt $max_timeout ]; then + timeout=$((timeout * 2)) fi done -echo "All topics are now available. Ready to go!" +tries=10 +timeout=1 +while true; do + if wget --spider -q "${CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL}/subjects" 2>/dev/null; then + break + fi + tries=$((tries - 1)) + if [ $tries -eq 0 ]; then + echo "FAILED TO REACH SCHEMA REGISTRY." + exit 6 + fi + echo "Failed to reach schema registry. Retrying in ${timeout} seconds." + sleep $timeout + if [ $timeout -lt $max_timeout ]; then + timeout=$((timeout * 2)) + fi +done + + +echo "Kafka is available. Ready to go!" diff --git a/images/radar-mongodb-connector/Dockerfile b/images/radar-mongodb-connector/Dockerfile index bad4d8699..3f39be7cb 100644 --- a/images/radar-mongodb-connector/Dockerfile +++ b/images/radar-mongodb-connector/Dockerfile @@ -11,19 +11,14 @@ # limitations under the License. -FROM confluentinc/cp-kafka-connect:3.1.2-1 +FROM confluentinc/cp-kafka-connect:3.2.1 MAINTAINER Nivethika M , Joris B , Francesco N LABEL description="RADAR-CNS Backend- MongoDB Sink Connector" -# Install RADAR-Backend -RUN echo "==> Installing Components" \ - # Download Git RADAR-Backend release - && echo "==> Downloading RADAR-CNS/RADAR-MongoDB-Sink-Connector 0.1-alpha.1 release from GitHub" - # Deploy RADAR-Backend - Mongodb Sink Connector -ADD https://github.com/RADAR-CNS/RADAR-MongoDB-Sink-Connector/releases/download/0.1-alpha.1/radar-mongodb-sink-connector-0.1-alpha.1.jar /etc/kafka-connect/jars/ +ADD https://github.com/RADAR-CNS/RADAR-MongoDB-Sink-Connector/releases/download/0.1/radar-mongodb-sink-connector-0.1.jar /etc/kafka-connect/jars/ # Load topics validator COPY ./kafka_status.sh /home/kafka_status.sh diff --git a/images/radar-mongodb-connector/kafka_status.sh b/images/radar-mongodb-connector/kafka_status.sh index b3e50697b..e488b1eef 100755 --- a/images/radar-mongodb-connector/kafka_status.sh +++ b/images/radar-mongodb-connector/kafka_status.sh @@ -2,56 +2,62 @@ # Check if variables exist if [ -z "$CONNECT_ZOOKEEPER_CONNECT" ]; then - echo "CONNECT_ZOOKEEPER_CONNECT is not defined" - exit 2 + echo "CONNECT_ZOOKEEPER_CONNECT is not defined" + exit 2 fi -if [ -z "$TOPIC_LIST" ]; then - echo "TOPIC_LIST is not defined" - exit 2 +if [ -z "$CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL" ]; then + echo "CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL is not defined" + exit 4 fi -# Save current IFS -SAVEIFS=$IFS +KAFKA_BROKERS=${KAFKA_BROKERS:-3} -# Fetch env topic list -IFS=', ' read -r -a needed <<< $TOPIC_LIST +max_timeout=32 -# Fetch env topic list -IFS=$'\n' -count=0 -interval=1 -max_retryes=15 -while [ "$count" != "${#needed[@]}" ] ; do +tries=10 +timeout=1 +while true; do + ZOOKEEPER_CHECK=$(zookeeper-shell $CONNECT_ZOOKEEPER_CONNECT <<< "ls /brokers/ids") + ZOOKEEPER_CHECK="${ZOOKEEPER_CHECK##*$'\n'}" + ZOOKEEPER_CHECK="$(echo -e "${ZOOKEEPER_CHECK}" | tr -d '[:space:]' | tr -d '[' | tr -d ']')" - if [ "$max_retryes" -eq "0" ] ; then - IFS=$SAVEIFS - echo "Force rebooting ... " - exit 2 + IFS=',' read -r -a array <<< $ZOOKEEPER_CHECK + LENGTH=${#array[@]} + if [ "$LENGTH" -eq "$KAFKA_BROKERS" ]; then + echo "Kafka brokers available." + break fi - count=0 - topics=$(kafka-topics --list --zookeeper $CONNECT_ZOOKEEPER_CONNECT) - topics=($topics) - - for topic in "${topics[@]}" - do - for need in "${needed[@]}" - do - if [ "$topic" = "$need" ] ; then - ((count++)) - fi - done - done - - if [ "$count" != "${#needed[@]}" ] ; then - echo "Waiting $interval second before retrying ..." - sleep $interval - if (( interval < 30 )); then - ((interval=interval*2)) - fi - ((max_retryes--)) + tries=$((tries - 1)) + if [ $tries -eq 0 ]; then + echo "FAILED: KAFKA BROKERs NOT READY." + exit 5 + fi + echo "Expected $KAFKA_BROKERS brokers but found only $LENGTH. Waiting $timeout second before retrying ..." + sleep $timeout + if [ $timeout -lt $max_timeout ]; then + timeout=$((timeout * 2)) fi done -echo "All topics are now available. Ready to go!" +tries=10 +timeout=1 +while true; do + if wget --spider -q "${CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL}/subjects" 2>/dev/null; then + break + fi + tries=$((tries - 1)) + if [ $tries -eq 0 ]; then + echo "FAILED TO REACH SCHEMA REGISTRY." + exit 6 + fi + echo "Failed to reach schema registry. Retrying in ${timeout} seconds." + sleep $timeout + if [ $timeout -lt $max_timeout ]; then + timeout=$((timeout * 2)) + fi +done + + +echo "Kafka is available. Ready to go!" diff --git a/images/radar-restapi/Dockerfile b/images/radar-restapi/Dockerfile index c511f6282..c83cc7d9d 100644 --- a/images/radar-restapi/Dockerfile +++ b/images/radar-restapi/Dockerfile @@ -13,6 +13,8 @@ FROM tomcat:8.0.44-jre8 +ENV JAVA_OPTS=-Djava.security.egd=file:/dev/urandom + MAINTAINER @fnobilia, @afolarin LABEL description="RADAR-CNS Rest Api docker container" diff --git a/scripts/restracture_backup_hdfs.sh b/scripts/restructure_backup_hdfs.sh similarity index 100% rename from scripts/restracture_backup_hdfs.sh rename to scripts/restructure_backup_hdfs.sh