From ceb8242c64c9ce5d66b81fb4ae1732d0bb28366c Mon Sep 17 00:00:00 2001 From: Christina Cortland Date: Thu, 14 Nov 2024 11:11:46 -0500 Subject: [PATCH 1/8] Output rails log to stdout in dev docker, remove unused debug_log config --- config/environments/development.rb | 7 ++++++- config/environments/integration.rb | 1 - config/environments/production.rb | 1 - config/environments/staging.rb | 1 - docker/run_dev.sh | 3 +++ 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/config/environments/development.rb b/config/environments/development.rb index cccc96a1..31e5688c 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -71,7 +71,12 @@ config.active_job.queue_adapter = :sidekiq # config.active_job.queue_name_prefix = "exhibits_#{Rails.env}" - config.debug_logger = Logger.new(Rails.root.join('log', 'debug.log')) + + if ENV["RAILS_LOG_TO_STDOUT"].present? + logger = ActiveSupport::Logger.new(STDOUT) + logger.formatter = ::Logger::Formatter.new + config.logger = ActiveSupport::TaggedLogging.new(logger) + end # provides access to IRB console on exception pages # config.web_console.whitelisted_ips = ENV['IP_WHITELIST'] if ENV['IP_WHITELIST'].present? diff --git a/config/environments/integration.rb b/config/environments/integration.rb index c7f64e07..1a6155b0 100644 --- a/config/environments/integration.rb +++ b/config/environments/integration.rb @@ -94,7 +94,6 @@ logger.formatter = config.log_formatter config.logger = ActiveSupport::TaggedLogging.new(logger) end - config.debug_logger = Logger.new(Rails.root.join('log', 'debug.log')) # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false diff --git a/config/environments/production.rb b/config/environments/production.rb index 31fe49e8..23c12fce 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -96,7 +96,6 @@ logger.formatter = config.log_formatter config.logger = ActiveSupport::TaggedLogging.new(logger) end - config.debug_logger = Logger.new(Rails.root.join('log', 'debug.log')) # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false diff --git a/config/environments/staging.rb b/config/environments/staging.rb index d5feee6a..6a9c4364 100644 --- a/config/environments/staging.rb +++ b/config/environments/staging.rb @@ -94,7 +94,6 @@ logger.formatter = config.log_formatter config.logger = ActiveSupport::TaggedLogging.new(logger) end - config.debug_logger = Logger.new(Rails.root.join('log', 'debug.log')) # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false diff --git a/docker/run_dev.sh b/docker/run_dev.sh index 72ee267d..dc5ec795 100755 --- a/docker/run_dev.sh +++ b/docker/run_dev.sh @@ -2,6 +2,9 @@ set -e +# Direct logs to stdout +export RAILS_LOG_TO_STDOUT="1" + # If the database exists, migrate. Otherwise setup (create and migrate) echo "Preparing Database..." bundle exec rake db:migrate 2>/dev/null || bundle exec rake db:environment:set RAILS_ENV=development db:create db:schema:load From 0e5767f7153783a36a3f9f400c1f3e9c636c491c Mon Sep 17 00:00:00 2001 From: Christina Cortland Date: Thu, 14 Nov 2024 11:13:05 -0500 Subject: [PATCH 2/8] Set up separate sidekiq service in dev docker --- compose.yaml | 11 +++++++++++ docker/run_dev.sh | 3 --- docker/run_sidekiq.sh | 9 +++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) create mode 100755 docker/run_sidekiq.sh diff --git a/compose.yaml b/compose.yaml index 2ae1c7f9..0922deba 100644 --- a/compose.yaml +++ b/compose.yaml @@ -22,6 +22,7 @@ services: - mysql - redis - solr + - sidekiq mysql: image: mariadb:10.6 @@ -55,6 +56,16 @@ services: volumes: - redis:/data + sidekiq: + build: + context: . + target: development + entrypoint: docker/run_sidekiq.sh + depends_on: + - redis + env_file: + - .env + volumes: db-data: gems: diff --git a/docker/run_dev.sh b/docker/run_dev.sh index dc5ec795..85cc418b 100755 --- a/docker/run_dev.sh +++ b/docker/run_dev.sh @@ -10,9 +10,6 @@ echo "Preparing Database..." bundle exec rake db:migrate 2>/dev/null || bundle exec rake db:environment:set RAILS_ENV=development db:create db:schema:load echo "Database Migration Done!" -# Start sidekiq -bundle exec sidekiq -L log/sidekiq.log -e $RAILS_ENV -C config/sidekiq.yml -d - # Start the web server mkdir -p ./tmp/pids bundle exec puma -C config/puma.rb -e $RAILS_ENV diff --git a/docker/run_sidekiq.sh b/docker/run_sidekiq.sh new file mode 100755 index 00000000..98e7f72d --- /dev/null +++ b/docker/run_sidekiq.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e + +# Start sidekiq +bundle exec sidekiq -e $RAILS_ENV -C config/sidekiq.yml + +# Run commands +exec "$@" From a986ef276908157920886d3c18467bde76289f31 Mon Sep 17 00:00:00 2001 From: Christina Cortland Date: Thu, 14 Nov 2024 13:50:42 -0500 Subject: [PATCH 3/8] Set up separate sidekiq service in integration docker --- Dockerfile | 1 + compose.integration.yaml | 13 +++++++++++++ docker/run.sh | 7 ++----- docker/run_sidekiq.sh | 1 + 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 31dadc95..e609124b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,6 +78,7 @@ WORKDIR ${APP_PATH} COPY ./Gemfile ./Gemfile.lock ./ RUN bundle config set --local with "${RAILS_ENV}" && \ bundle config set --local without "${BUNDLE_WITHOUT}" && \ + bundle config set --local path "${BUNDLE_PATH}" && \ bundle install && \ gem install aws-sdk-s3 && \ rm -rf ${BUNDLE_PATH}/cache/*.gem && \ diff --git a/compose.integration.yaml b/compose.integration.yaml index 13cb522b..e52e212f 100644 --- a/compose.integration.yaml +++ b/compose.integration.yaml @@ -18,3 +18,16 @@ services: - PORT=9292 ports: - 9292:9292 + depends_on: + - sidekiq + + sidekiq: + build: + context: . + args: + BUNDLE_WITHOUT: "development test" + RACK_ENV: integration + RAILS_ENV: integration + entrypoint: docker/run_sidekiq.sh + env_file: + - .env diff --git a/docker/run.sh b/docker/run.sh index 5e39dc83..1ff55407 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -17,12 +17,9 @@ then fi # Run db migrations -echo "Preparing Database..." +echo "Preparing database..." bundle exec rake db:migrate RAILS_ENV=$RAILS_ENV -echo "Database Migration Done!" - -# Start sidekiq -bundle exec sidekiq -L log/sidekiq.log -e $RAILS_ENV -C config/sidekiq.yml -d +echo "Database migration done!" # Start the web server mkdir -p ./tmp/pids diff --git a/docker/run_sidekiq.sh b/docker/run_sidekiq.sh index 98e7f72d..85bfe57c 100755 --- a/docker/run_sidekiq.sh +++ b/docker/run_sidekiq.sh @@ -3,6 +3,7 @@ set -e # Start sidekiq +echo "Starting up sidekiq..." bundle exec sidekiq -e $RAILS_ENV -C config/sidekiq.yml # Run commands From 1ba76926f9831f220e37d7fe9028b59fbba5b816 Mon Sep 17 00:00:00 2001 From: Christina Cortland Date: Thu, 14 Nov 2024 13:51:05 -0500 Subject: [PATCH 4/8] Move assets precompile to docker run script --- Dockerfile | 2 +- docker/run.sh | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e609124b..05c36fe8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -86,7 +86,7 @@ RUN bundle config set --local with "${RAILS_ENV}" && \ find ${BUNDLE_PATH}/ -name "*.o" -delete COPY . . -RUN bundle exec rake assets:precompile && rm .env +RUN rm .env ################################################################################ # Final image for integration/staging/production diff --git a/docker/run.sh b/docker/run.sh index 1ff55407..a5942f2a 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -21,6 +21,11 @@ echo "Preparing database..." bundle exec rake db:migrate RAILS_ENV=$RAILS_ENV echo "Database migration done!" +# Precompile assets +echo "Compiling assets..." +bundle exec rake assets:precompile +echo "Compiling assets done!" + # Start the web server mkdir -p ./tmp/pids bundle exec puma -C config/puma.rb -e $RAILS_ENV From 8cafa249fc44a2e81ed4512ef18d9ab1a6d50e3e Mon Sep 17 00:00:00 2001 From: Christina Cortland Date: Thu, 14 Nov 2024 13:51:57 -0500 Subject: [PATCH 5/8] Run useradd -m option in dockerfile --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 05c36fe8..4ad1cbb7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,9 +103,7 @@ ENV RAILS_ENV=${RAILS_ENV} \ # exhibits_cron - .ebextentions/tmp_cleanup.config # localtime adjustment - .ebextentions/system_time.config COPY ./cron/exhibits_cron /etc/cron.d/exhibits_cron -RUN groupadd -r $GROUP && useradd -r -g $GROUP $USER && \ - mkdir -p /home/${USER} && \ - chown ${USER}:${GROUP} /home/${USER} && \ +RUN groupadd -r $GROUP && useradd -mr -g $GROUP $USER && \ chmod gu+rw /var/run && chmod gu+s /usr/sbin/cron && \ crontab -u root /etc/cron.d/exhibits_cron && \ ln -sf /usr/share/zoneinfo/America/New_York /etc/localtime From 1cc3a380482ef430204692389f08138218730702 Mon Sep 17 00:00:00 2001 From: Christina Cortland Date: Thu, 14 Nov 2024 16:09:39 -0500 Subject: [PATCH 6/8] Extract out sidekiq service for prod and stg docker --- compose.production.yaml | 13 +++++++++++++ compose.staging.yaml | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/compose.production.yaml b/compose.production.yaml index 2e303bdb..a2e28519 100644 --- a/compose.production.yaml +++ b/compose.production.yaml @@ -16,3 +16,16 @@ services: - PORT=9292 ports: - 9292:9292 + depends_on: + - sidekiq + + sidekiq: + build: + context: . + args: + BUNDLE_WITHOUT: "development integration test" + RACK_ENV: production + RAILS_ENV: production + entrypoint: docker/run_sidekiq.sh + env_file: + - .env diff --git a/compose.staging.yaml b/compose.staging.yaml index 8c8417e7..93bc2a09 100644 --- a/compose.staging.yaml +++ b/compose.staging.yaml @@ -16,3 +16,16 @@ services: - PORT=9292 ports: - 9292:9292 + depends_on: + - sidekiq + + sidekiq: + build: + context: . + args: + BUNDLE_WITHOUT: "development test" + RACK_ENV: staging + RAILS_ENV: staging + entrypoint: docker/run_sidekiq.sh + env_file: + - .env From 228c2b985a010b5a471f8c1d8866980846f567af Mon Sep 17 00:00:00 2001 From: Christina Cortland Date: Fri, 22 Nov 2024 12:00:24 -0500 Subject: [PATCH 7/8] Get .env from s3 if not available when running sidekiq docker script --- docker/run_sidekiq.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker/run_sidekiq.sh b/docker/run_sidekiq.sh index 85bfe57c..c8559ae7 100755 --- a/docker/run_sidekiq.sh +++ b/docker/run_sidekiq.sh @@ -2,6 +2,13 @@ set -e +# Get environment variables +WORKDIR="/exhibits" +ENVFILE="${WORKDIR}/.env" +if [ ! -f "$ENVFILE" ]; then + ruby docker/get_env.rb $RAILS_ENV $WORKDIR +fi + # Start sidekiq echo "Starting up sidekiq..." bundle exec sidekiq -e $RAILS_ENV -C config/sidekiq.yml From 89f3d64ad7e9131bcf51c4eea105f64562375c62 Mon Sep 17 00:00:00 2001 From: Christina Cortland Date: Wed, 27 Nov 2024 11:08:14 -0500 Subject: [PATCH 8/8] Update comments in compose files --- compose.integration.yaml | 5 ++++- compose.production.yaml | 2 +- compose.staging.yaml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/compose.integration.yaml b/compose.integration.yaml index e52e212f..4fd8d5cf 100644 --- a/compose.integration.yaml +++ b/compose.integration.yaml @@ -1,5 +1,8 @@ # Docker compose file for integration environment -# Run with: docker compose up +# Run with: docker compose -f compose.integration.yaml up +# +# For local development testing of the integration container: +# consider setting up a redis container, instead of connecting to the remote redis host. services: webapp: diff --git a/compose.production.yaml b/compose.production.yaml index a2e28519..851dbfbc 100644 --- a/compose.production.yaml +++ b/compose.production.yaml @@ -1,5 +1,5 @@ # Docker compose file for production environment -# Run with: docker compose up +# Run with: docker compose -f compose.production.yaml up services: webapp: diff --git a/compose.staging.yaml b/compose.staging.yaml index 93bc2a09..c26520f6 100644 --- a/compose.staging.yaml +++ b/compose.staging.yaml @@ -1,5 +1,5 @@ # Docker compose file for staging environment -# Run with: docker compose up +# Run with: docker compose -f compose.production.yaml up services: webapp: