From d3fe2c67a2556cf5e931f0841e5f015f1d85f5be Mon Sep 17 00:00:00 2001 From: Colin Craig Date: Wed, 13 Jan 2016 12:18:44 -0500 Subject: [PATCH 1/5] Added initial Docker setup for ES --- Dockerfile | 94 ++++++++++++++++++++++++++++++++++++++++++ api/config.js | 41 +++--------------- fbopen-docker-setup.sh | 71 +++++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+), 36 deletions(-) create mode 100644 Dockerfile create mode 100755 fbopen-docker-setup.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..80bae15 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,94 @@ +FROM ubuntu:trusty + +#Adapted from https://github.com/18F/docker-elasticsearch + +# To run: +# $: docker build -t esfbopen . + +# docker run -p 9200:9200 -p 9300:9300 esfbopen \ + # && --rm \ + # && --name es-test \ + # && -v /home/fbopen:/data + +# To ssh into docker cont. -> +# $: docker exec -i -t bash + +MAINTAINER Colin Craig + +RUN \ + apt-get update \ + -qq \ + && apt-get install \ + -qq \ + --yes \ + --no-install-recommends \ + --no-install-suggests \ + curl \ + python-software-properties \ + software-properties-common \ + +# Clean up packages. + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + + +# Add Java. +RUN \ + echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections \ + && add-apt-repository -y ppa:webupd8team/java \ + && apt-get update \ + -qq \ + && apt-get install \ + -qq \ + -y oracle-java7-installer=7u80+7u60arm-0~webupd8~1 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /var/cache/oracle-jdk7-installer + +# Define commonly used JAVA_HOME variable +ENV JAVA_HOME /usr/lib/jvm/java-7-oracle + +# Install Elasticsearch. +ENV ES_PKG_NAME elasticsearch-1.7.1 +RUN \ + cd / \ + && wget https://download.elasticsearch.org/elasticsearch/elasticsearch/${ES_PKG_NAME}.tar.gz \ + --no-verbose \ + && tar xvzf $ES_PKG_NAME.tar.gz \ + && rm -f $ES_PKG_NAME.tar.gz \ + && mv /$ES_PKG_NAME /elasticsearch + +# Set up default config. +ADD elasticsearch/elasticsearch__dev.yml /elasticsearch/elasticsearch.yml + +# mapper-attachments plugin. +ENV MA_VERSION=2.7.0 +RUN /elasticsearch/bin/plugin \ + install elasticsearch/elasticsearch-mapper-attachments/${MA_VERSION} \ + --silent + +# elasticsearch-cloud-aws plugin. +ENV CA_VERSION=2.7.0 +RUN /elasticsearch/bin/plugin install \ + elasticsearch/elasticsearch-cloud-aws/${CA_VERSION} \ + --silent + + +# Create the Elasticsearch user. +RUN groupadd -r elasticsearch \ + && useradd -r -g elasticsearch elasticsearch + +# Use a reasonable heap size. +# https://www.elastic.co/guide/en/elasticsearch/guide/current/heap-sizing.html +ENV ES_HEAP_SIZE=1g + +# Mount for persistent data. +VOLUME ["/data"] +WORKDIR /data + +# Expose Elasticsearch ports. +EXPOSE 9200 9300 + +# Entry commands. +#ENTRYPOINT ["/"] +CMD ["/elasticsearch/bin/elasticsearch"] diff --git a/api/config.js b/api/config.js index b25a7f8..e9dab0d 100644 --- a/api/config.js +++ b/api/config.js @@ -1,5 +1,4 @@ // Create config.js, in this directory, based on this sample file. -var appEnv = require('cfenv').getAppEnv(); fs = require('fs'); @@ -14,54 +13,24 @@ config.app.read_only = true; // require basic authentication config.app.require_http_basic_auth = false; -config.app.http_basic_auth = { - realm: 'fbopen' - , file: '/path/to/htpasswd_file' // username/password file created using htpasswd -}; // http config.app.listen_http = true; -config.app.port = process.env.PORT || 3000; +config.app.port = process.env.FBOPEN_API_PORT || 3000; // https config.app.listen_https = false; // logger config.logger = {}; -config.logger.path = __dirname + '/../log/api.log'; - -// settings from cloud foundry env -cf_services = JSON.parse((process.env.VCAP_SERVICES || '{}')); +config.logger.path = '/var/log/fbopen/api.log'; // elasticsearch -// henceforth, all ES service instances shall be named "es-YYYYMMDD", -// with the date being the day they were instantiated -config.elasticsearch = appEnv.getServiceCreds("es-.*"); -// the above defines the following keys: [ hostname, password, port, uri, username ] - -// if running locally -if (!config.elasticsearch) { - config.elasticsearch = { - uri: 'http://localhost:9200', - hostname: 'localhost', - port: 9200 - }; -} - -console.log('[API] Discovered elasticsearch config: ', config.elasticsearch); - +config.elasticsearch = {}; +config.elasticsearch.host = process.env.ELASTICSEARCH_HOST || 'localhost'; +config.elasticsearch.port = '9200'; config.elasticsearch.index = process.env.ELASTICSEARCH_INDEX || 'fbopen'; // this is used to adjust the date math that's used for searching within the (statically-dated) test data config.elasticsearch.now_str = process.env.ELASTICSEARCH_NOW || 'now'; -// Sentry config -if (process.env.env !== 'development') { - config.sentry_uri = appEnv.getServiceURL('sentry'); - - if (!config.sentry_uri) { - console.error('The Sentry URI could not be found.'); - process.exit(1); - } -} - module.exports = config; diff --git a/fbopen-docker-setup.sh b/fbopen-docker-setup.sh new file mode 100755 index 0000000..d6444cd --- /dev/null +++ b/fbopen-docker-setup.sh @@ -0,0 +1,71 @@ +# http://stackoverflow.com/a/3931779/94154 +command_exists () { + type "$1" &> /dev/null ; +} + +# Build our Docker ES image. +docker build -t es-fbopen . + +# Run Docker ES image. +# Run as Daemon, expose ports 9200:9300, sync /home/fbopen dir to /data +docker run -d -it -p 9200:9200 -p 9300:9300 -v /home/fbopen:/data esfbopen + +# Forward Docker containter ES ports to Host localhost 9200-9300 range. +VBoxManage controlvm $(docker-machine active) natpf1 "name,tcp,127.0.0.1,9200,,9300" + +# Do git stuff. +git submodule update --init --recursive +npm i -g mocha elasticdump json + +# API +echo "Creating api/config.js from sample." +cp api/config-sample_dev.js api/config.js +mkdir -p log/ +touch log/api.log + +# init index with mappings and settings on ES Image. +curl -XPUT -v $(docker-machine ip $(docker-machine active)):9200/fbopen0 --data-binary elasticsearch/init.json + +if command_exists node ; then + echo +else + echo "It looks like node.js is not installed. Please install it." + exit 1 +fi + +cd api +npm install +cd .. + +echo "Starting node API server" +osascript< Date: Wed, 13 Jan 2016 12:31:21 -0500 Subject: [PATCH 2/5] Pulled config.js back from CF branch. Not sure why this file isnt in a .gitignore. --- api/config.js | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/api/config.js b/api/config.js index e9dab0d..b25a7f8 100644 --- a/api/config.js +++ b/api/config.js @@ -1,4 +1,5 @@ // Create config.js, in this directory, based on this sample file. +var appEnv = require('cfenv').getAppEnv(); fs = require('fs'); @@ -13,24 +14,54 @@ config.app.read_only = true; // require basic authentication config.app.require_http_basic_auth = false; +config.app.http_basic_auth = { + realm: 'fbopen' + , file: '/path/to/htpasswd_file' // username/password file created using htpasswd +}; // http config.app.listen_http = true; -config.app.port = process.env.FBOPEN_API_PORT || 3000; +config.app.port = process.env.PORT || 3000; // https config.app.listen_https = false; // logger config.logger = {}; -config.logger.path = '/var/log/fbopen/api.log'; +config.logger.path = __dirname + '/../log/api.log'; + +// settings from cloud foundry env +cf_services = JSON.parse((process.env.VCAP_SERVICES || '{}')); // elasticsearch -config.elasticsearch = {}; -config.elasticsearch.host = process.env.ELASTICSEARCH_HOST || 'localhost'; -config.elasticsearch.port = '9200'; +// henceforth, all ES service instances shall be named "es-YYYYMMDD", +// with the date being the day they were instantiated +config.elasticsearch = appEnv.getServiceCreds("es-.*"); +// the above defines the following keys: [ hostname, password, port, uri, username ] + +// if running locally +if (!config.elasticsearch) { + config.elasticsearch = { + uri: 'http://localhost:9200', + hostname: 'localhost', + port: 9200 + }; +} + +console.log('[API] Discovered elasticsearch config: ', config.elasticsearch); + config.elasticsearch.index = process.env.ELASTICSEARCH_INDEX || 'fbopen'; // this is used to adjust the date math that's used for searching within the (statically-dated) test data config.elasticsearch.now_str = process.env.ELASTICSEARCH_NOW || 'now'; +// Sentry config +if (process.env.env !== 'development') { + config.sentry_uri = appEnv.getServiceURL('sentry'); + + if (!config.sentry_uri) { + console.error('The Sentry URI could not be found.'); + process.exit(1); + } +} + module.exports = config; From 839212841b6f65e6f85c07d48649e3315399bb31 Mon Sep 17 00:00:00 2001 From: Colin Craig Date: Wed, 13 Jan 2016 13:36:58 -0500 Subject: [PATCH 3/5] Added readme, renamed file --- README.md | 8 +++++ fbopen-docker-setup.sh | 71 ------------------------------------------ 2 files changed, 8 insertions(+), 71 deletions(-) delete mode 100755 fbopen-docker-setup.sh diff --git a/README.md b/README.md index 3d52003..98beb45 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,14 @@ To clean out any new files created from that script, as well as uninstall Elasti $ ./initial-dev-uninstall.sh ``` +### Quickstart (Requires Docker) (experimental) + +```sh +$ cd fbopen +$ ./fbopen-docker-setup.sh +``` + + ### How to get started (manually) * Clone this repo. * This repo has an external dependency on another git repo, which needs to be populated at first, so `cd` to the repo and run: `git submodule update --init --recursive`. diff --git a/fbopen-docker-setup.sh b/fbopen-docker-setup.sh deleted file mode 100755 index d6444cd..0000000 --- a/fbopen-docker-setup.sh +++ /dev/null @@ -1,71 +0,0 @@ -# http://stackoverflow.com/a/3931779/94154 -command_exists () { - type "$1" &> /dev/null ; -} - -# Build our Docker ES image. -docker build -t es-fbopen . - -# Run Docker ES image. -# Run as Daemon, expose ports 9200:9300, sync /home/fbopen dir to /data -docker run -d -it -p 9200:9200 -p 9300:9300 -v /home/fbopen:/data esfbopen - -# Forward Docker containter ES ports to Host localhost 9200-9300 range. -VBoxManage controlvm $(docker-machine active) natpf1 "name,tcp,127.0.0.1,9200,,9300" - -# Do git stuff. -git submodule update --init --recursive -npm i -g mocha elasticdump json - -# API -echo "Creating api/config.js from sample." -cp api/config-sample_dev.js api/config.js -mkdir -p log/ -touch log/api.log - -# init index with mappings and settings on ES Image. -curl -XPUT -v $(docker-machine ip $(docker-machine active)):9200/fbopen0 --data-binary elasticsearch/init.json - -if command_exists node ; then - echo -else - echo "It looks like node.js is not installed. Please install it." - exit 1 -fi - -cd api -npm install -cd .. - -echo "Starting node API server" -osascript< Date: Wed, 13 Jan 2016 13:38:10 -0500 Subject: [PATCH 4/5] Added readme, renamed file --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 98beb45..1dcd118 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ $ ./initial-dev-uninstall.sh ```sh $ cd fbopen -$ ./fbopen-docker-setup.sh +$ ./fbo-docker-setup.sh ``` From d6bacc4a3376b1ba08fbd593ea8e75dc335f9353 Mon Sep 17 00:00:00 2001 From: Colin Craig Date: Wed, 13 Jan 2016 13:39:11 -0500 Subject: [PATCH 5/5] Forgot to add file --- fbo-docker-setup.sh | 71 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100755 fbo-docker-setup.sh diff --git a/fbo-docker-setup.sh b/fbo-docker-setup.sh new file mode 100755 index 0000000..26c54e7 --- /dev/null +++ b/fbo-docker-setup.sh @@ -0,0 +1,71 @@ +# http://stackoverflow.com/a/3931779/94154 +command_exists () { + type "$1" &> /dev/null ; +} + +# Build our Docker ES image. +docker build -t es-fbopen . + +# Run Docker ES image. +# Run as Daemon, expose ports 9200:9300, sync /home/fbopen dir to /data +docker run -d -it -p 9200:9200 -p 9300:9300 -v /home/fbopen:/data esfbopen + +# Forward Docker containter ES ports to Host localhost 9200-9300 range. +VBoxManage controlvm $(docker-machine active) natpf1 "name,tcp,127.0.0.1,9200,,9300" + +# Do git stuff. +git submodule update --init --recursive +npm i -g mocha elasticdump json + +# API +#echo "Creating api/config.js from sample." +#cp api/config-sample_dev.js api/config.js +mkdir -p log/ +touch log/api.log + +# init index with mappings and settings on ES Image. +curl -XPUT -v $(docker-machine ip $(docker-machine active)):9200/fbopen0 --data-binary elasticsearch/init.json + +if command_exists node ; then + echo +else + echo "It looks like node.js is not installed. Please install it." + exit 1 +fi + +cd api +npm install +cd .. + +echo "Starting node API server" +osascript<