From dc2499997f45465276d6364ac9bfab8a1f013049 Mon Sep 17 00:00:00 2001 From: Niteesh Date: Sat, 17 Jun 2017 02:02:01 +0530 Subject: [PATCH 1/3] Starting only particular process --- Dockerfile | 15 ++- broker-supervisord.conf | 19 +++ .../druid/_common/common.runtime.properties | 118 ++++++++++++++++++ conf-quickstart/druid/_common/log4j2.xml | 33 +++++ conf-quickstart/druid/broker/jvm.config | 8 ++ .../druid/broker/runtime.properties | 16 +++ conf-quickstart/druid/coordinator/jvm.config | 8 ++ .../druid/coordinator/runtime.properties | 5 + conf-quickstart/druid/historical/jvm.config | 8 ++ .../druid/historical/runtime.properties | 13 ++ .../druid/middleManager/jvm.config | 7 ++ .../druid/middleManager/runtime.properties | 20 +++ conf-quickstart/druid/overlord/jvm.config | 7 ++ .../druid/overlord/runtime.properties | 7 ++ conf-quickstart/tranquility/kafka.json | 77 ++++++++++++ conf-quickstart/tranquility/server.json | 74 +++++++++++ coordinator-supervisord.conf | 31 +++++ historical-supervisord.conf | 26 ++++ middleManager-supervisord.conf | 20 +++ mysql-supervisord.conf | 8 ++ overlord-supervisord.conf | 20 +++ start.sh | 41 ++++++ supervisord.conf | 72 ----------- zookeeper-supervisord.conf | 10 ++ 24 files changed, 590 insertions(+), 73 deletions(-) create mode 100644 broker-supervisord.conf create mode 100644 conf-quickstart/druid/_common/common.runtime.properties create mode 100644 conf-quickstart/druid/_common/log4j2.xml create mode 100644 conf-quickstart/druid/broker/jvm.config create mode 100644 conf-quickstart/druid/broker/runtime.properties create mode 100644 conf-quickstart/druid/coordinator/jvm.config create mode 100644 conf-quickstart/druid/coordinator/runtime.properties create mode 100644 conf-quickstart/druid/historical/jvm.config create mode 100644 conf-quickstart/druid/historical/runtime.properties create mode 100644 conf-quickstart/druid/middleManager/jvm.config create mode 100644 conf-quickstart/druid/middleManager/runtime.properties create mode 100644 conf-quickstart/druid/overlord/jvm.config create mode 100644 conf-quickstart/druid/overlord/runtime.properties create mode 100644 conf-quickstart/tranquility/kafka.json create mode 100644 conf-quickstart/tranquility/server.json create mode 100644 coordinator-supervisord.conf create mode 100644 historical-supervisord.conf create mode 100644 middleManager-supervisord.conf create mode 100644 mysql-supervisord.conf create mode 100644 overlord-supervisord.conf create mode 100644 start.sh create mode 100644 zookeeper-supervisord.conf diff --git a/Dockerfile b/Dockerfile index 0307e15..c6f6c6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,6 +41,7 @@ RUN mkdir -p /usr/local/druid/lib # trigger rebuild only if branch changed ADD https://api.github.com/repos/$GITHUB_OWNER/druid/git/refs/heads/$DRUID_VERSION druid-version.json RUN git clone -q --branch $DRUID_VERSION --depth 1 https://github.com/$GITHUB_OWNER/druid.git /tmp/druid +ADD config-quickstart /tmp/druid/examples/conf-quickstart WORKDIR /tmp/druid # package and install Druid locally @@ -76,6 +77,18 @@ RUN /etc/init.d/mysql start \ # Setup supervisord ADD supervisord.conf /etc/supervisor/conf.d/supervisord.conf +ADD broker-supervisord.conf /etc/supervisor/conf.d/broker-supervisord.conf +ADD coordinator-supervisord.conf /etc/supervisor/conf.d/coordinator-supervisord.conf +ADD historical-supervisord.conf /etc/supervisor/conf.d/historical-supervisord.conf +ADD middleManager-supervisord.conf /etc/supervisor/conf.d/middleManager-supervisord.conf +ADD mysql-supervisord.conf /etc/supervisor/conf.d/mysql-supervisord.conf +ADD overlord-supervisord.conf /etc/supervisor/conf.d/overlord-supervisord.conf +ADD zookeeper-supervisord.conf /etc/supervisor/conf.d/zookeeper-supervisord.conf + +#setup start script +ADD start.sh /bin/start +RUN chmod a+x /bin/start + # Expose ports: # - 8081: HTTP (coordinator) @@ -92,4 +105,4 @@ EXPOSE 3306 EXPOSE 2181 2888 3888 WORKDIR /var/lib/druid -ENTRYPOINT export HOSTIP="$(resolveip -s $HOSTNAME)" && exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf +ENTRYPOINT start all diff --git a/broker-supervisord.conf b/broker-supervisord.conf new file mode 100644 index 0000000..98f3f6e --- /dev/null +++ b/broker-supervisord.conf @@ -0,0 +1,19 @@ + +; +; config for broker +; + +[program:druid-broker] +user=druid +command=java + -server + -Xmx1g + -Duser.timezone=UTC + -Dfile.encoding=UTF-8 + -Ddruid.host=%(ENV_HOSTIP)s + -Ddruid.computation.buffer.size=67108864 + -Ddruid.broker.cache.sizeInBytes=33554432 + -cp /usr/local/druid/lib/* + io.druid.cli.Main server broker +redirect_stderr=true +priority=100 diff --git a/conf-quickstart/druid/_common/common.runtime.properties b/conf-quickstart/druid/_common/common.runtime.properties new file mode 100644 index 0000000..ba73feb --- /dev/null +++ b/conf-quickstart/druid/_common/common.runtime.properties @@ -0,0 +1,118 @@ +# +# Licensed to Metamarkets Group Inc. (Metamarkets) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. Metamarkets licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Extensions specified in the load list will be loaded by Druid +# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead +# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead + +# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. +# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. +# More info: http://druid.io/docs/latest/operations/including-extensions.html +druid.extensions.loadList=[] + +# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory +# and uncomment the line below to point to your directory. +#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + +# +# Logging +# + +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties=true + +# +# Zookeeper +# + +druid.zk.service.host=localhost +druid.zk.paths.base=/druid + +# +# Metadata storage +# + +# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): +druid.metadata.storage.type=derby +druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +druid.metadata.storage.connector.host=localhost +druid.metadata.storage.connector.port=1527 + +# For MySQL: +#druid.metadata.storage.type=mysql +#druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# For PostgreSQL: +#druid.metadata.storage.type=postgresql +#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# +# Deep storage +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.storage.type=local +druid.storage.storageDirectory=var/druid/segments + +# For HDFS: +#druid.storage.type=hdfs +#druid.storage.storageDirectory=/druid/segments + +# For S3: +#druid.storage.type=s3 +#druid.storage.bucket=your-bucket +#druid.storage.baseKey=druid/segments +#druid.s3.accessKey=... +#druid.s3.secretKey=... + +# +# Indexing service logs +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.indexer.logs.type=file +druid.indexer.logs.directory=var/druid/indexing-logs + +# For HDFS: +#druid.indexer.logs.type=hdfs +#druid.indexer.logs.directory=/druid/indexing-logs + +# For S3: +#druid.indexer.logs.type=s3 +#druid.indexer.logs.s3Bucket=your-bucket +#druid.indexer.logs.s3Prefix=druid/indexing-logs + +# +# Service discovery +# + +druid.selectors.indexing.serviceName=druid/overlord +druid.selectors.coordinator.serviceName=druid/coordinator + +# +# Monitoring +# + +druid.monitoring.monitors=["com.metamx.metrics.JvmMonitor"] +druid.emitter=logging +druid.emitter.logging.logLevel=info diff --git a/conf-quickstart/druid/_common/log4j2.xml b/conf-quickstart/druid/_common/log4j2.xml new file mode 100644 index 0000000..a60ebda --- /dev/null +++ b/conf-quickstart/druid/_common/log4j2.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + diff --git a/conf-quickstart/druid/broker/jvm.config b/conf-quickstart/druid/broker/jvm.config new file mode 100644 index 0000000..608aa35 --- /dev/null +++ b/conf-quickstart/druid/broker/jvm.config @@ -0,0 +1,8 @@ +-server +-Xms1g +-Xmx1g +-XX:MaxDirectMemorySize=1792m +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf-quickstart/druid/broker/runtime.properties b/conf-quickstart/druid/broker/runtime.properties new file mode 100644 index 0000000..bb2aaba --- /dev/null +++ b/conf-quickstart/druid/broker/runtime.properties @@ -0,0 +1,16 @@ +druid.service=druid/broker +druid.port=8082 + +# HTTP server threads +druid.broker.http.numConnections=5 +druid.server.http.numThreads=9 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=256000000 +druid.processing.numThreads=2 + +# Query cache (we use a small local cache) +druid.broker.cache.useCache=true +druid.broker.cache.populateCache=true +druid.cache.type=local +druid.cache.sizeInBytes=10000000 diff --git a/conf-quickstart/druid/coordinator/jvm.config b/conf-quickstart/druid/coordinator/jvm.config new file mode 100644 index 0000000..66aa951 --- /dev/null +++ b/conf-quickstart/druid/coordinator/jvm.config @@ -0,0 +1,8 @@ +-server +-Xms256m +-Xmx256m +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-Dderby.stream.error.file=var/druid/derby.log diff --git a/conf-quickstart/druid/coordinator/runtime.properties b/conf-quickstart/druid/coordinator/runtime.properties new file mode 100644 index 0000000..a61482f --- /dev/null +++ b/conf-quickstart/druid/coordinator/runtime.properties @@ -0,0 +1,5 @@ +druid.service=druid/coordinator +druid.port=8081 + +druid.coordinator.startDelay=PT10S +druid.coordinator.period=PT5S diff --git a/conf-quickstart/druid/historical/jvm.config b/conf-quickstart/druid/historical/jvm.config new file mode 100644 index 0000000..430209e --- /dev/null +++ b/conf-quickstart/druid/historical/jvm.config @@ -0,0 +1,8 @@ +-server +-Xms1g +-Xmx1g +-XX:MaxDirectMemorySize=1280m +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf-quickstart/druid/historical/runtime.properties b/conf-quickstart/druid/historical/runtime.properties new file mode 100644 index 0000000..8d8633d --- /dev/null +++ b/conf-quickstart/druid/historical/runtime.properties @@ -0,0 +1,13 @@ +druid.service=druid/historical +druid.port=8083 + +# HTTP server threads +druid.server.http.numThreads=9 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=256000000 +druid.processing.numThreads=2 + +# Segment storage +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize"\:300000000000}] +druid.server.maxSize=300000000000 diff --git a/conf-quickstart/druid/middleManager/jvm.config b/conf-quickstart/druid/middleManager/jvm.config new file mode 100644 index 0000000..7b9f1a4 --- /dev/null +++ b/conf-quickstart/druid/middleManager/jvm.config @@ -0,0 +1,7 @@ +-server +-Xms64m +-Xmx64m +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf-quickstart/druid/middleManager/runtime.properties b/conf-quickstart/druid/middleManager/runtime.properties new file mode 100644 index 0000000..dc1f6ac --- /dev/null +++ b/conf-quickstart/druid/middleManager/runtime.properties @@ -0,0 +1,20 @@ +druid.service=druid/middleManager +druid.port=8091 + +# Number of tasks per middleManager +druid.worker.capacity=3 + +# Task launch parameters +druid.indexer.runner.javaOpts=-server -Xmx2g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.task.baseTaskDir=var/druid/task + +# HTTP server threads +druid.server.http.numThreads=9 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=256000000 +druid.processing.numThreads=2 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp +druid.indexer.task.defaultHadoopCoordinates=["org.apache.hadoop:hadoop-client:2.7.3"] diff --git a/conf-quickstart/druid/overlord/jvm.config b/conf-quickstart/druid/overlord/jvm.config new file mode 100644 index 0000000..90526a5 --- /dev/null +++ b/conf-quickstart/druid/overlord/jvm.config @@ -0,0 +1,7 @@ +-server +-Xms256m +-Xmx256m +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf-quickstart/druid/overlord/runtime.properties b/conf-quickstart/druid/overlord/runtime.properties new file mode 100644 index 0000000..7ab544a --- /dev/null +++ b/conf-quickstart/druid/overlord/runtime.properties @@ -0,0 +1,7 @@ +druid.service=druid/overlord +druid.port=8090 + +druid.indexer.queue.startDelay=PT5S + +druid.indexer.runner.type=remote +druid.indexer.storage.type=metadata diff --git a/conf-quickstart/tranquility/kafka.json b/conf-quickstart/tranquility/kafka.json new file mode 100644 index 0000000..ffe5378 --- /dev/null +++ b/conf-quickstart/tranquility/kafka.json @@ -0,0 +1,77 @@ +{ + "dataSources" : { + "metrics-kafka" : { + "spec" : { + "dataSchema" : { + "dataSource" : "metrics-kafka", + "parser" : { + "type" : "string", + "parseSpec" : { + "timestampSpec" : { + "column" : "timestamp", + "format" : "auto" + }, + "dimensionsSpec" : { + "dimensions" : [], + "dimensionExclusions" : [ + "timestamp", + "value" + ] + }, + "format" : "json" + } + }, + "granularitySpec" : { + "type" : "uniform", + "segmentGranularity" : "hour", + "queryGranularity" : "none" + }, + "metricsSpec" : [ + { + "type" : "count", + "name" : "count" + }, + { + "name" : "value_sum", + "type" : "doubleSum", + "fieldName" : "value" + }, + { + "fieldName" : "value", + "name" : "value_min", + "type" : "doubleMin" + }, + { + "type" : "doubleMax", + "name" : "value_max", + "fieldName" : "value" + } + ] + }, + "ioConfig" : { + "type" : "realtime" + }, + "tuningConfig" : { + "type" : "realtime", + "maxRowsInMemory" : "100000", + "intermediatePersistPeriod" : "PT10M", + "windowPeriod" : "PT10M" + } + }, + "properties" : { + "task.partitions" : "1", + "task.replicants" : "1", + "topicPattern" : "metrics" + } + } + }, + "properties" : { + "zookeeper.connect" : "localhost", + "druid.discovery.curator.path" : "/druid/discovery", + "druid.selectors.indexing.serviceName" : "druid/overlord", + "commit.periodMillis" : "15000", + "consumer.numThreads" : "2", + "kafka.zookeeper.connect" : "localhost", + "kafka.group.id" : "tranquility-kafka" + } +} diff --git a/conf-quickstart/tranquility/server.json b/conf-quickstart/tranquility/server.json new file mode 100644 index 0000000..cbc1d14 --- /dev/null +++ b/conf-quickstart/tranquility/server.json @@ -0,0 +1,74 @@ +{ + "dataSources" : { + "metrics" : { + "spec" : { + "dataSchema" : { + "dataSource" : "metrics", + "parser" : { + "type" : "string", + "parseSpec" : { + "timestampSpec" : { + "column" : "timestamp", + "format" : "auto" + }, + "dimensionsSpec" : { + "dimensions" : [], + "dimensionExclusions" : [ + "timestamp", + "value" + ] + }, + "format" : "json" + } + }, + "granularitySpec" : { + "type" : "uniform", + "segmentGranularity" : "hour", + "queryGranularity" : "none" + }, + "metricsSpec" : [ + { + "type" : "count", + "name" : "count" + }, + { + "name" : "value_sum", + "type" : "doubleSum", + "fieldName" : "value" + }, + { + "fieldName" : "value", + "name" : "value_min", + "type" : "doubleMin" + }, + { + "type" : "doubleMax", + "name" : "value_max", + "fieldName" : "value" + } + ] + }, + "ioConfig" : { + "type" : "realtime" + }, + "tuningConfig" : { + "type" : "realtime", + "maxRowsInMemory" : "100000", + "intermediatePersistPeriod" : "PT10M", + "windowPeriod" : "PT10M" + } + }, + "properties" : { + "task.partitions" : "1", + "task.replicants" : "1" + } + } + }, + "properties" : { + "zookeeper.connect" : "localhost", + "druid.discovery.curator.path" : "/druid/discovery", + "druid.selectors.indexing.serviceName" : "druid/overlord", + "http.port" : "8200", + "http.threads" : "9" + } +} diff --git a/coordinator-supervisord.conf b/coordinator-supervisord.conf new file mode 100644 index 0000000..8d2a125 --- /dev/null +++ b/coordinator-supervisord.conf @@ -0,0 +1,31 @@ +; +; config for coordinator +; + +[program:druid-coordinator] +user=druid +command=java + -server + -Xmx1g + -Duser.timezone=UTC + -Dfile.encoding=UTF-8 + -Ddruid.host=%(ENV_HOSTIP)s + -Ddruid.extensions.loadList=[\"mysql-metadata-storage\"] + -Ddruid.extensions.directory=/usr/local/druid/extensions + -Ddruid.extensions.hadoopDependenciesDir=/usr/local/druid/hadoop-dependencies + -Ddruid.metadata.storage.type=mysql + -Ddruid.metadata.storage.connector.connectURI=jdbc:mysql://localhost:3306/druid + -Ddruid.metadata.storage.connector.user=druid + -Ddruid.metadata.storage.connector.password=diurd + -Ddruid.coordinator.asOverlord.enabled=true + -Ddruid.coordinator.asOverlord.overlordService=druid/overlord + -Ddruid.indexer.fork.property.druid.processing.numThreads=1 + -Ddruid.indexer.storage.type=metadata + -Ddruid.indexer.queue.startDelay=PT0M + -Ddruid.indexer.runner.javaOpts="-server -Xmx1g -XX:MaxDirectMemorySize=2147483648" + -Ddruid.processing.buffer.sizeBytes=536870912 + -Ddruid.coordinator.startDelay=PT5S + -cp /usr/local/druid/lib/* + io.druid.cli.Main server coordinator +redirect_stderr=true +priority=100 \ No newline at end of file diff --git a/historical-supervisord.conf b/historical-supervisord.conf new file mode 100644 index 0000000..5c5efad --- /dev/null +++ b/historical-supervisord.conf @@ -0,0 +1,26 @@ + +; +; config for historical +; + + +[program:druid-historical] +user=druid +command=java + -server + -Xmx1g + -Duser.timezone=UTC + -Dfile.encoding=UTF-8 + -Ddruid.host=%(ENV_HOSTIP)s + -Ddruid.extensions.loadList=[\"druid-s3-extensions\"] + -Ddruid.extensions.directory=/usr/local/druid/extensions + -Ddruid.extensions.hadoopDependenciesDir=/usr/local/druid/hadoop-dependencies + -Ddruid.s3.accessKey=AKIAIMKECRUYKDQGR6YQ + -Ddruid.s3.secretKey=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 + -Ddruid.computation.buffer.size=67108864 + -Ddruid.segmentCache.locations="[{\"path\":\"/var/tmp/druid/indexCache\",\"maxSize\":5000000000}]" + -Ddruid.server.maxSize=5000000000 + -cp /usr/local/druid/lib/* + io.druid.cli.Main server historical +redirect_stderr=true +priority=100 diff --git a/middleManager-supervisord.conf b/middleManager-supervisord.conf new file mode 100644 index 0000000..7b32bb9 --- /dev/null +++ b/middleManager-supervisord.conf @@ -0,0 +1,20 @@ +; +; config for middleManager +; + + + +[program:druid-middleManager] +user=druid +command=java + -server + -Xms64m + -Xmx64m + -Duser.timezone=UTC + -Dfile.encoding=UTF-8 + -Djava.io.tmpdir=var/tmp + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + -cp /usr/local/druid/lib/* + io.druid.cli.Main server middleManager +redirect_stderr=true +priority=100 diff --git a/mysql-supervisord.conf b/mysql-supervisord.conf new file mode 100644 index 0000000..fd6862a --- /dev/null +++ b/mysql-supervisord.conf @@ -0,0 +1,8 @@ +; +; config for mysql +; + +[program:mysql] +command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe +user=mysql +priority=0 \ No newline at end of file diff --git a/overlord-supervisord.conf b/overlord-supervisord.conf new file mode 100644 index 0000000..36beeef --- /dev/null +++ b/overlord-supervisord.conf @@ -0,0 +1,20 @@ + +; +; config for overlord +; + + +[program:druid-overlord] +user=druid +command=java + -server + -Xms256m + -Xmx256m + -Duser.timezone=UTC + -Dfile.encoding=UTF-8 + -Djava.io.tmpdir=var/tmp + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + -cp /usr/local/druid/lib/* + io.druid.cli.Main server overlord +redirect_stderr=true +priority=100 diff --git a/start.sh b/start.sh new file mode 100644 index 0000000..dca1876 --- /dev/null +++ b/start.sh @@ -0,0 +1,41 @@ +#!/bin/bash +case "$1" in + historical) + echo "STARTING HISTORICAL" + cat /etc/supervisor/conf.d/supervisord.conf /etc/supervisor/conf.d/historical-supervisord.conf > /etc/supervisor/conf.d/run-supervisord.conf + ;; + broker) + echo "STARTING BROKER" + cat /etc/supervisor/conf.d/supervisord.conf /etc/supervisor/conf.d/broker-supervisord.conf > /etc/supervisor/conf.d/run-supervisord.conf + ;; + coordinator) + echo "STARTING CORDINATOR" + cat /etc/supervisor/conf.d/supervisord.conf /etc/supervisor/conf.d/coordinator-supervisord.conf > /etc/supervisor/conf.d/run-supervisord.conf + ;; + middleManager) + echo "STARTING MIDDLEMANAGER" + cat /etc/supervisor/conf.d/supervisord.conf /etc/supervisor/conf.d/middleManager-supervisord.conf > /etc/supervisor/conf.d/run-supervisord.conf + ;; + overlord) + echo "STARTING OVERLORD" + cat /etc/supervisor/conf.d/supervisord.conf /etc/supervisor/conf.d/overlord-supervisord.conf > /etc/supervisor/conf.d/run-supervisord.conf + ;; + zookeeper) + echo "STARTING OVERLORD" + cat /etc/supervisor/conf.d/supervisord.conf /etc/supervisor/conf.d/zookeeper-supervisord.conf > /etc/supervisor/conf.d/run-supervisord.conf + ;; + mysql) + echo "STARTING MYSQL" + cat /etc/supervisor/conf.d/supervisord.conf /etc/supervisor/conf.d/mysql-supervisord.conf > /etc/supervisor/conf.d/run-supervisord.conf + ;; + all) + echo "STARTING DRUID" + cat /etc/supervisor/conf.d/supervisord.conf /etc/supervisor/conf.d/zookeeper-supervisord.conf /etc/supervisor/conf.d/mysql-supervisord.conf /etc/supervisor/conf.d/historical-supervisord.conf /etc/supervisor/conf.d/broker-supervisord.conf /etc/supervisor/conf.d/coordinator-supervisord.conf /etc/supervisor/conf.d/middleManager-supervisord.conf > /etc/supervisor/conf.d/run-supervisord.conf + ;; + *) + echo "Usage: $NAME {historical|broker|coordinator|middleManager|overlord}" >&2 + exit 1 + ;; +esac + +export HOSTIP="$(resolveip -s $HOSTNAME)" && exec /usr/bin/supervisord -c /etc/supervisor/conf.d/run-supervisord.conf \ No newline at end of file diff --git a/supervisord.conf b/supervisord.conf index aeeff7b..b842245 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -2,76 +2,4 @@ nodaemon=true loglevel=debug -[program:zookeeper] -command=/usr/local/zookeeper/bin/zkServer.sh start-foreground -user=daemon -priority=0 -[program:mysql] -command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe -user=mysql -priority=0 - -[program:druid-coordinator] -user=druid -command=java - -server - -Xmx1g - -Duser.timezone=UTC - -Dfile.encoding=UTF-8 - -Ddruid.host=%(ENV_HOSTIP)s - -Ddruid.extensions.loadList=[\"mysql-metadata-storage\"] - -Ddruid.extensions.directory=/usr/local/druid/extensions - -Ddruid.extensions.hadoopDependenciesDir=/usr/local/druid/hadoop-dependencies - -Ddruid.metadata.storage.type=mysql - -Ddruid.metadata.storage.connector.connectURI=jdbc:mysql://localhost:3306/druid - -Ddruid.metadata.storage.connector.user=druid - -Ddruid.metadata.storage.connector.password=diurd - -Ddruid.coordinator.asOverlord.enabled=true - -Ddruid.coordinator.asOverlord.overlordService=druid/overlord - -Ddruid.indexer.fork.property.druid.processing.numThreads=1 - -Ddruid.indexer.storage.type=metadata - -Ddruid.indexer.queue.startDelay=PT0M - -Ddruid.indexer.runner.javaOpts="-server -Xmx1g -XX:MaxDirectMemorySize=2147483648" - -Ddruid.processing.buffer.sizeBytes=536870912 - -Ddruid.coordinator.startDelay=PT5S - -cp /usr/local/druid/lib/* - io.druid.cli.Main server coordinator -redirect_stderr=true -priority=100 - -[program:druid-historical] -user=druid -command=java - -server - -Xmx1g - -Duser.timezone=UTC - -Dfile.encoding=UTF-8 - -Ddruid.host=%(ENV_HOSTIP)s - -Ddruid.extensions.loadList=[\"druid-s3-extensions\"] - -Ddruid.extensions.directory=/usr/local/druid/extensions - -Ddruid.extensions.hadoopDependenciesDir=/usr/local/druid/hadoop-dependencies - -Ddruid.s3.accessKey=AKIAIMKECRUYKDQGR6YQ - -Ddruid.s3.secretKey=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 - -Ddruid.computation.buffer.size=67108864 - -Ddruid.segmentCache.locations="[{\"path\":\"/var/tmp/druid/indexCache\",\"maxSize\":5000000000}]" - -Ddruid.server.maxSize=5000000000 - -cp /usr/local/druid/lib/* - io.druid.cli.Main server historical -redirect_stderr=true -priority=100 - -[program:druid-broker] -user=druid -command=java - -server - -Xmx1g - -Duser.timezone=UTC - -Dfile.encoding=UTF-8 - -Ddruid.host=%(ENV_HOSTIP)s - -Ddruid.computation.buffer.size=67108864 - -Ddruid.broker.cache.sizeInBytes=33554432 - -cp /usr/local/druid/lib/* - io.druid.cli.Main server broker -redirect_stderr=true -priority=100 diff --git a/zookeeper-supervisord.conf b/zookeeper-supervisord.conf new file mode 100644 index 0000000..974a182 --- /dev/null +++ b/zookeeper-supervisord.conf @@ -0,0 +1,10 @@ +; +; config for zookeeper +; + + + +[program:zookeeper] +command=/usr/local/zookeeper/bin/zkServer.sh start-foreground +user=daemon +priority=0 \ No newline at end of file From 5d9f7124c3250960c1dfe3c80592085bd0e3f5ce Mon Sep 17 00:00:00 2001 From: Niteesh Date: Sun, 18 Jun 2017 00:52:38 +0530 Subject: [PATCH 2/3] Starting only particular process --- Dockerfile | 18 ++++++++++++------ historical-supervisord.conf | 6 ++++++ middleManager-supervisord.conf | 7 +++++++ mysql-supervisord.conf | 9 ++++++++- overlord-supervisord.conf | 8 ++++++++ start.sh | 1 + zookeeper-supervisord.conf | 7 ++++++- 7 files changed, 48 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index c6f6c6d..c6ec612 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,6 @@ FROM ubuntu:14.04 -# Set version and github repo which you want to build from -ENV GITHUB_OWNER druid-io -ENV DRUID_VERSION 0.9.2 -ENV ZOOKEEPER_VERSION 3.4.9 + # Java 8 RUN apt-get update \ @@ -25,11 +22,17 @@ RUN wget -q -O - http://archive.apache.org/dist/maven/maven-3/3.2.5/binaries/apa && ln -s /usr/local/apache-maven-3.2.5 /usr/local/apache-maven \ && ln -s /usr/local/apache-maven/bin/mvn /usr/local/bin/mvn + + + # Zookeeper +ENV ZOOKEEPER_VERSION 3.4.9 RUN wget -q -O - http://www.us.apache.org/dist/zookeeper/zookeeper-$ZOOKEEPER_VERSION/zookeeper-$ZOOKEEPER_VERSION.tar.gz | tar -xzf - -C /usr/local \ && cp /usr/local/zookeeper-$ZOOKEEPER_VERSION/conf/zoo_sample.cfg /usr/local/zookeeper-$ZOOKEEPER_VERSION/conf/zoo.cfg \ && ln -s /usr/local/zookeeper-$ZOOKEEPER_VERSION /usr/local/zookeeper + + # Druid system user RUN adduser --system --group --no-create-home druid \ && mkdir -p /var/lib/druid \ @@ -38,10 +41,13 @@ RUN adduser --system --group --no-create-home druid \ # Druid (from source) RUN mkdir -p /usr/local/druid/lib +# Set version and github repo which you want to build from +ENV GITHUB_OWNER druid-io +ENV DRUID_VERSION 0.10.1 # trigger rebuild only if branch changed ADD https://api.github.com/repos/$GITHUB_OWNER/druid/git/refs/heads/$DRUID_VERSION druid-version.json RUN git clone -q --branch $DRUID_VERSION --depth 1 https://github.com/$GITHUB_OWNER/druid.git /tmp/druid -ADD config-quickstart /tmp/druid/examples/conf-quickstart +ADD conf-quickstart /tmp/druid/examples/conf-quickstart WORKDIR /tmp/druid # package and install Druid locally @@ -105,4 +111,4 @@ EXPOSE 3306 EXPOSE 2181 2888 3888 WORKDIR /var/lib/druid -ENTRYPOINT start all +ENTRYPOINT /bin/start all diff --git a/historical-supervisord.conf b/historical-supervisord.conf index 5c5efad..0a4cf7c 100644 --- a/historical-supervisord.conf +++ b/historical-supervisord.conf @@ -24,3 +24,9 @@ command=java io.druid.cli.Main server historical redirect_stderr=true priority=100 + + +; +;config end for historical +; + diff --git a/middleManager-supervisord.conf b/middleManager-supervisord.conf index 7b32bb9..11bf6a3 100644 --- a/middleManager-supervisord.conf +++ b/middleManager-supervisord.conf @@ -1,3 +1,4 @@ + ; ; config for middleManager ; @@ -18,3 +19,9 @@ command=java io.druid.cli.Main server middleManager redirect_stderr=true priority=100 + +; +; config END for middleManager +; + + diff --git a/mysql-supervisord.conf b/mysql-supervisord.conf index fd6862a..f681acc 100644 --- a/mysql-supervisord.conf +++ b/mysql-supervisord.conf @@ -5,4 +5,11 @@ [program:mysql] command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe user=mysql -priority=0 \ No newline at end of file +priority=0 + + +; +; config for mysql +; + + diff --git a/overlord-supervisord.conf b/overlord-supervisord.conf index 36beeef..e23e47c 100644 --- a/overlord-supervisord.conf +++ b/overlord-supervisord.conf @@ -18,3 +18,11 @@ command=java io.druid.cli.Main server overlord redirect_stderr=true priority=100 + + + +; +; config END for mysql +; + + diff --git a/start.sh b/start.sh index dca1876..86a4cb3 100644 --- a/start.sh +++ b/start.sh @@ -1,4 +1,5 @@ #!/bin/bash + case "$1" in historical) echo "STARTING HISTORICAL" diff --git a/zookeeper-supervisord.conf b/zookeeper-supervisord.conf index 974a182..d2f8d3a 100644 --- a/zookeeper-supervisord.conf +++ b/zookeeper-supervisord.conf @@ -7,4 +7,9 @@ [program:zookeeper] command=/usr/local/zookeeper/bin/zkServer.sh start-foreground user=daemon -priority=0 \ No newline at end of file +priority=0 + + +; +; config END for zookeeper +; \ No newline at end of file From 7c10da72e527cba16600a66686cebcdcfc550421 Mon Sep 17 00:00:00 2001 From: Niteesh Date: Sun, 18 Jun 2017 02:01:06 +0530 Subject: [PATCH 3/3] Starting only particular process --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index c6ec612..f6cfbe6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,5 @@ FROM ubuntu:14.04 - - # Java 8 RUN apt-get update \ && apt-get install -y software-properties-common \ @@ -111,4 +109,4 @@ EXPOSE 3306 EXPOSE 2181 2888 3888 WORKDIR /var/lib/druid -ENTRYPOINT /bin/start all +CMD /bin/start all