Raise key error if env variable is nil & update docker.env (#597)

* Replace `ENV.fetch(..., nil)` by `ENV.fetch(...)`

This means that if the env variable is not set, we will encounter an exception
which is good as otherwise we'd just ignore that exception even though it
might result in serious repercussions.

Regex used for matching (in VSCode): ENV.fetch\(".*",\snil\)

* Sync `docker.env` with actual file on server

* Replace `ENV[]` by `ENV.fetch(...)` whenever applicable

Sometimes `ENV[]` is used in a statement where we check if the env variable
is present or not. In this case, we did not change it to `ENV.fetch(...)`.

* Fix missing quotes around env variable

Error we got during deployment:
docker.env: line 24: Users/mampf: No such file or directory

* Source dummy `docker.env` file (workaround)

See the comment in the code for an in-depth explanation for this
workaround.

* Replace `<>` syntax in env file to allow sourcing it

* Copy dummy docker file into container

* Use automatic export of env variables

by setting `set -o allexport` (or equivalently: `set -a`).

* Use new env syntax for registration limit check

* Fall back to `nil` as was before for `RAILS_CACHE_ID`

app/controllers/erdbeere_controller.rb

@@ -8,7 +8,7 @@ def current_ability
   end
 
   def show_example
-    response = Faraday.get(ENV.fetch("ERDBEERE_API", nil) + "/examples/#{params[:id]}")
+    response = Faraday.get(ENV.fetch("ERDBEERE_API") + "/examples/#{params[:id]}")
     @content = if response.status == 200
       JSON.parse(response.body)["embedded_html"]
     else
@@ -17,7 +17,7 @@ def show_example
   end
 
   def show_property
-    response = Faraday.get(ENV.fetch("ERDBEERE_API", nil) + "/properties/#{params[:id]}")
+    response = Faraday.get(ENV.fetch("ERDBEERE_API") + "/properties/#{params[:id]}")
 
     @content = if response.status == 200
       JSON.parse(response.body)["embedded_html"]
@@ -28,7 +28,7 @@ def show_property
 
   def show_structure
     params[:id]
-    response = Faraday.get(ENV.fetch("ERDBEERE_API", nil) + "/structures/#{params[:id]}")
+    response = Faraday.get(ENV.fetch("ERDBEERE_API") + "/structures/#{params[:id]}")
     @content = if response.status == 200
       JSON.parse(response.body)["embedded_html"]
     else
@@ -51,7 +51,7 @@ def cancel_edit_tags
   def display_info
     @id = params[:id]
     @sort = params[:sort]
-    response = Faraday.get(ENV.fetch("ERDBEERE_API", nil) +
+    response = Faraday.get(ENV.fetch("ERDBEERE_API") +
                            "/#{@sort.downcase.pluralize}/#{@id}/view_info")
     @content = JSON.parse(response.body)
     if response.status != 200
@@ -87,7 +87,7 @@ def update_tags
   end
 
   def fill_realizations_select
-    response = Faraday.get("#{ENV.fetch("ERDBEERE_API", nil)}/structures/")
+    response = Faraday.get("#{ENV.fetch("ERDBEERE_API")}/structures/")
     @tag = Tag.find_by(id: params[:id])
     hash = JSON.parse(response.body)
     @structures = hash["data"].map do |d|
@@ -102,7 +102,7 @@ def fill_realizations_select
   end
 
   def find_example
-    response = Faraday.get("#{ENV.fetch("ERDBEERE_API", nil)}/find?#{find_params.to_query}")
+    response = Faraday.get("#{ENV.fetch("ERDBEERE_API")}/find?#{find_params.to_query}")
     @content = if response.status == 200
       JSON.parse(response.body)["embedded_html"]
     else

app/controllers/lectures_controller.rb

@@ -217,7 +217,7 @@ def edit_structures
 
   def search_examples
     if @lecture.structure_ids.any?
-      response = Faraday.get("#{ENV.fetch("ERDBEERE_API", nil)}/search")
+      response = Faraday.get("#{ENV.fetch("ERDBEERE_API")}/search")
       @form = JSON.parse(response.body)["embedded_html"]
       # rubocop:disable Style/StringConcatenation
       @form.gsub!("token_placeholder",
@@ -402,7 +402,7 @@ def eager_load_stuff
 
     def set_erdbeere_data
       @structure_ids = @lecture.structure_ids
-      response = Faraday.get("#{ENV.fetch("ERDBEERE_API", nil)}/structures")
+      response = Faraday.get("#{ENV.fetch("ERDBEERE_API")}/structures")
       response_hash = if response.status == 200
         JSON.parse(response.body)
       else

app/controllers/registrations_controller.rb

@@ -9,12 +9,12 @@ def verify_captcha
     return true unless ENV["USE_CAPTCHA_SERVICE"]
 
     begin
-      uri = URI.parse(ENV.fetch("CAPTCHA_VERIFY_URL", nil))
+      uri = URI.parse(ENV.fetch("CAPTCHA_VERIFY_URL"))
       data = { message: params["frc-captcha-solution"],
-               application_token: ENV.fetch("CAPTCHA_APPLICATION_TOKEN", nil) }
+               application_token: ENV.fetch("CAPTCHA_APPLICATION_TOKEN") }
       header = { "Content-Type": "text/json" }
       http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = true if ENV["CAPTCHA_VERIFY_URL"].include?("https")
+      http.use_ssl = true if ENV.fetch("CAPTCHA_VERIFY_URL").include?("https")
       request = Net::HTTP::Post.new(uri.request_uri, header)
       request.body = data.to_json
 
@@ -70,9 +70,9 @@ def after_sign_up_path_for(_resource)
   private
 
     def check_registration_limit
-      timeframe = ((ENV["MAMPF_REGISTRATION_TIMEFRAME"] || 15).to_i.minutes.ago..)
+      timeframe = (ENV.fetch("MAMPF_REGISTRATION_TIMEFRAME", 15).to_i.minutes.ago..)
       num_new_registrations = User.where(confirmed_at: nil, created_at: timeframe).count
-      max_registrations = (ENV["MAMPF_MAX_REGISTRATION_PER_TIMEFRAME"] || 40).to_i
+      max_registrations = ENV.fetch("MAMPF_MAX_REGISTRATION_PER_TIMEFRAME", 40).to_i
       return if num_new_registrations <= max_registrations
 
       # Current number of new registrations is too high

app/helpers/application_helper.rb

@@ -16,7 +16,7 @@ def current_lecture
   def host
     if Rails.env.production?
       # rubocop:disable Style/StringConcatenation
-      ENV.fetch("MEDIA_SERVER", nil) + "/" + ENV.fetch("INSTANCE_NAME", nil)
+      ENV.fetch("MEDIA_SERVER") + "/" + ENV.fetch("INSTANCE_NAME")
       # rubocop:enable Style/StringConcatenation
     else
       ""
@@ -29,7 +29,7 @@ def host
   # the actual media server.
   # This is used for the download buttons for videos and manuscripts.
   def download_host
-    Rails.env.production? ? ENV.fetch("DOWNLOAD_LOCATION", nil) : ""
+    Rails.env.production? ? ENV.fetch("DOWNLOAD_LOCATION") : ""
   end
 
   # Returns the full title on a per-page basis.

app/mailers/exception_handler/exception_mailer.rb

@@ -5,7 +5,7 @@ class ExceptionMailer < ApplicationMailer
 
     # Defaults
     default subject: I18n.t("exception.exception",
-                            host: ENV.fetch("URL_HOST", nil))
+                            host: ENV.fetch("URL_HOST"))
     default from: ExceptionHandler.config.email
     default template_path: "exception_handler/mailers"
     # => http://stackoverflow.com/a/18579046/1143732

app/models/user_cleaner.rb

@@ -3,9 +3,9 @@ class UserCleaner
   attr_accessor :imap, :email_dict, :hash_dict
 
   def login
-    @imap = Net::IMAP.new(ENV.fetch("IMAPSERVER", nil), port: 993, ssl: true)
-    @imap.authenticate("LOGIN", ENV.fetch("PROJECT_EMAIL_USERNAME", nil),
-                       ENV.fetch("PROJECT_EMAIL_PASSWORD", nil))
+    @imap = Net::IMAP.new(ENV.fetch("IMAPSERVER"), port: 993, ssl: true)
+    @imap.authenticate("LOGIN", ENV.fetch("PROJECT_EMAIL_USERNAME"),
+                       ENV.fetch("PROJECT_EMAIL_PASSWORD"))
   end
 
   def logout
@@ -15,7 +15,7 @@ def logout
   def search_emails_and_hashes
     @email_dict = {}
     @hash_dict = {}
-    @imap.examine(ENV.fetch("PROJECT_EMAIL_MAILBOX", nil))
+    @imap.examine(ENV.fetch("PROJECT_EMAIL_MAILBOX"))
     # Mails containing multiple email addresses (Subject: "Undelivered Mail Returned to Sender")
     @imap.search(["SUBJECT",
                   "Undelivered Mail Returned to Sender"]).each do |message_id|
@@ -103,7 +103,7 @@ def move_mail(message_ids, attempt = 0)
     return if attempt > 3
 
     begin
-      @imap.examine(ENV.fetch("PROJECT_EMAIL_MAILBOX", nil))
+      @imap.examine(ENV.fetch("PROJECT_EMAIL_MAILBOX"))
       @imap.move(message_ids, "Other Users/mampf/handled_bounces")
     rescue Net::IMAP::BadResponseError
       move_mail(message_ids, attempt + 1)

app/views/devise/registrations/new.html.erb

@@ -33,13 +33,13 @@
                                   target: :_blank)), class: "d-inline", for: "dsgvo-consent" %>
   </div>
   <%= f.hidden_field :locale, value: I18n.locale %>
-  <% if ENV['USE_CAPTCHA_SERVICE']%>
+  <% if ENV["USE_CAPTCHA_SERVICE"]%>
   <p>
-  <div id="captcha-widget"  data-sitekey="YOUR_SITE_KEY" data-start="auto" data-captcha-url="<%=ENV['CAPTCHA_PUZZLE_URL']%>" data-lang="<%=I18n.locale %>"></div>
+  <div id="captcha-widget"  data-sitekey="YOUR_SITE_KEY" data-start="auto" data-captcha-url="<%=ENV.fetch('CAPTCHA_PUZZLE_URL') %>" data-lang="<%=I18n.locale %>"></div>
   </p>
   <%end %>
   <div class="mb-3">
-    <%= f.submit t('.sign_up'), class: 'btn btn-primary', id: 'register-user', disabled:ENV['USE_CAPTCHA_SERVICE'] %>
+    <%= f.submit t('.sign_up'), class: 'btn btn-primary', id: 'register-user', disabled:ENV.fetch("USE_CAPTCHA_SERVICE") %>
   </div>
 <% end %>
 

app/views/exception_handler/mailers/new_exception.html.erb

@@ -1,5 +1,5 @@
 <%= t('exception.exception_report',
-			host: ENV['URL_HOST']) %>
+			host: ENV.fetch("URL_HOST")) %>
 <strong>
 	<%= @exception.response %> (<%= @exception.status %>)
 </strong>

app/workers/interaction_saver.rb

@@ -2,8 +2,8 @@ class InteractionSaver
   include Sidekiq::Worker
 
   def perform(session_id, full_path, referrer, study_participant)
-    referrer_url = if referrer.to_s.include?(ENV["URL_HOST"])
-      referrer.to_s.remove(ENV["URL_HOST"])
+    referrer_url = if referrer.to_s.include?(ENV.fetch("URL_HOST"))
+      referrer.to_s.remove(ENV.fetch("URL_HOST"))
               .remove("https://").remove("http://")
     end
     Interaction.create(session_id: Digest::SHA2.hexdigest(session_id).first(10),

config/application.rb

@@ -24,7 +24,7 @@ class Application < Rails::Application
     # the framework and any gems in your application.
     config.exception_handler = {
       # sends exception emails to a listed email (string // "[email protected]")
-      email: ENV.fetch("ERROR_EMAIL", nil),
+      email: ENV.fetch("ERROR_EMAIL"),
 
       # All keys interpolated as strings, so you can use
       # symbols, strings or integers where necessary

config/environments/production.rb

@@ -55,23 +55,23 @@
   config.log_tags = [:request_id]
 
   # Use a different cache store in production.
-  config.cache_store = :mem_cache_store, ENV.fetch("MEMCACHED_SERVER", nil)
+  config.cache_store = :mem_cache_store, ENV.fetch("MEMCACHED_SERVER")
 
   # Use a real queuing backend for Active Job (and separate queues per environment)
   # config.active_job.queue_adapter     = :resque
   # config.active_job.queue_name_prefix = "mampf_#{Rails.env}"
   config.action_mailer.perform_caching = false
-  config.action_mailer.default_url_options = { protocol: "https", host: ENV.fetch("URL_HOST", nil) }
+  config.action_mailer.default_url_options = { protocol: "https", host: ENV.fetch("URL_HOST") }
 
   config.action_mailer.delivery_method = :smtp
   config.action_mailer.perform_deliveries = true
   config.action_mailer.raise_delivery_errors = false
   config.action_mailer.default(charset: "utf-8")
 
   config.action_mailer.smtp_settings = {
-    address: ENV.fetch("MAILSERVER", nil),
+    address: ENV.fetch("MAILSERVER"),
     port: 25,
-    domain: ENV.fetch("MAILSERVER", nil)
+    domain: ENV.fetch("MAILSERVER")
   }
 
   # Ignore bad email addresses and do not raise email delivery errors.

config/initializers/default_setting.rb

@@ -1,10 +1,10 @@
 class DefaultSetting
-  ERDBEERE_LINK = ENV.fetch("ERDBEERE_SERVER", nil)
-  MUESLI_LINK = ENV.fetch("MUESLI_SERVER", nil)
-  PROJECT_EMAIL = ENV.fetch("PROJECT_EMAIL", nil)
-  PROJECT_NOTIFICATION_EMAIL = ENV.fetch("PROJECT_NOTIFICATION_EMAIL", nil)
-  BLOG_LINK = ENV.fetch("BLOG", nil)
-  URL_HOST_SHORT = ENV.fetch("URL_HOST_SHORT", nil)
+  ERDBEERE_LINK = ENV.fetch("ERDBEERE_SERVER")
+  MUESLI_LINK = ENV.fetch("MUESLI_SERVER")
+  PROJECT_EMAIL = ENV.fetch("PROJECT_EMAIL")
+  PROJECT_NOTIFICATION_EMAIL = ENV.fetch("PROJECT_NOTIFICATION_EMAIL")
+  BLOG_LINK = ENV.fetch("BLOG")
+  URL_HOST_SHORT = ENV.fetch("URL_HOST_SHORT")
   RESEARCHGATE_LINK = "https://www.researchgate.net/project/MaMpf-Mathematische-Medienplattform".freeze
   TOUR_LINK = "https://mampf.blog/ueber-mampf/".freeze
   RESOURCES_LINK = "https://mampf.blog/ressourcen-fur-editorinnen/".freeze

config/initializers/devise.rb

@@ -12,7 +12,7 @@
   # note that it will be overwritten if you use your own mailer class
   # with default "from" parameter.
   config.mailer_sender = if Rails.env.production?
-    ENV.fetch("FROM_ADDRESS", nil)
+    ENV.fetch("FROM_ADDRESS")
   else
     "[email protected]"
   end

config/initializers/thredded.rb

@@ -190,4 +190,4 @@
 #
 # add in (must install separate gem (under development) as well):
 # Thredded.notifiers = [Thredded::EmailNotifier.new,
-#                       Thredded::PushoverNotifier.new(ENV.fetch("PUSHOVER_APP_ID", nil))]
+#                       Thredded::PushoverNotifier.new(ENV.fetch("PUSHOVER_APP_ID"))]

docker/production/Dockerfile

@@ -81,5 +81,36 @@ RUN bundle install
 RUN yarn install --production=true
 
 COPY --chown=app:app . /usr/src/app
+
+# The command ". ./docker-dummy.env" will source our dummy docker env file.
+# So why do we need this?
+#
+# Well, (deeply inhales), Rails needs to boot entirely to run the
+# `assets:precompile` task. Therefore, it also needs to access the env variables
+# to correctly start the initializers.
+#
+# However (after a long ime researching), docker compose does not seem to offer
+# an easy solution to have an env file from the host machine available during
+# the build step (Dockerfile) and not just during the run time of the container.
+# Note that the env file is indeed available on our host, just not in the build
+# context, the latter being the MaMpf github repo that docker compose pulls from.
+#
+# Even with volumes and bind mounts it's not working properly ("file not found").
+# In the end, we found a solution that suggests to use the new docker buildkit
+# to allow for multiple build contexts. Yet, we explicitly set DOCKER_BUILDKIT=0
+# to use the old buildkit since the new one always gives a moby-related ssh error.
+# And even if this worked, it is not entirely clear if this is even working
+# with docker compose or just with docker (sigh).
+#
+# That's why, in the end, we decided to leverage our already-existing dummy env
+# file and source it here in the Dockerfile just to have the precompile task run
+# successfully (this task doesn't even rely on the actual values, so despite
+# being a hack, it should be fine).
+# 
+# I've written down more details in this question on StackOverflow:
+# https://stackoverflow.com/q/78098380/
+COPY ./docker/production/docker.env ./docker-dummy.env
+
 RUN cp -r $(bundle info --path sidekiq)/web/assets /usr/src/app/public/sidekiq && \
-    SECRET_KEY_BASE="$(bundle exec rails secret)" DB_ADAPTER=nulldb   bundle exec rails assets:precompile
+    set -o allexport && . ./docker-dummy.env && set +o allexport && \
+    SECRET_KEY_BASE="$(bundle exec rails secret)" DB_ADAPTER=nulldb bundle exec rails assets:precompile

docker/production/docker.env

@@ -1,4 +1,5 @@
-GIT_BRANCH=production
+# Instance variables
+GIT_BRANCH=main
 COMPOSE_PROJECT_NAME=mampf
 INSTANCE_NAME=mampf
 
@@ -14,15 +15,23 @@ MEMCACHED_SERVER=cache
 FROM_ADDRESS=[email protected]
 MAILSERVER=mail.mathi.uni-heidelberg.de
 PROJECT_EMAIL=[email protected]
-ERROR_EMAIL=[email protected]
+PROJECT_NOTIFICATION_EMAIL=notificationmail
 MAILID_DOMAIN=mathi.uni-heidelberg.de
+ERROR_EMAIL=[email protected]
 IMAPSERVER=mail.mathi.uni-heidelberg.de
 PROJECT_EMAIL_USERNAME=creativeusername
 PROJECT_EMAIL_PASSWORD=secretsecret
-PROJECT_EMAIL_MAILBOX=Other Users/mampf
+PROJECT_EMAIL_MAILBOX="Other Users/mampf"
 
 # Due to CORS constraints, some urls are proxied to the media server
 DOWNLOAD_LOCATION=https://mampf.mathi.uni-heidelberg.de/mediaforward
+REWRITE_ENABLED=1
+
+# Captcha Server
+USE_CAPTCHA_SERVICE=1
+CAPTCHA_VERIFY_URL=https://captcha2go.mathi.uni-heidelberg.de/v1/verify
+CAPTCHA_PUZZLE_URL=https://captcha2go.mathi.uni-heidelberg.de/v1/puzzle
+CAPTCHA_APPLICATION_TOKEN=secret
 
 # Upload folder
 MEDIA_PATH=/private/media
@@ -35,14 +44,14 @@ PRODUCTION_DATABASE_INTERACTIONS=mampf_interactions
 PRODUCTION_DATABASE_HOST=db
 PRODUCTION_DATABASE_USERNAME=mampf
 PRODUCTION_DATABASE_PASSWORD=supersecret
-PRODUCTION_DATABASE_PORT=5432
-PRODUCTION_DATABASE_URL='postgresql://mampf:supersecret@db:5432/mampf'
+PRODUCTION_DATABASE_PORT=port
+PRODUCTION_DATABASE_URL='postgresql://mampf:supersecret@db:port/mampf'
 
 # Rails configuration
 # change RAILS_ENV to production for a production deployment
 RAILS_ENV=production
-RAILS_MASTER_KEY=<rails master key>
-SECRET_KEY_BASE=<secret key base>
+RAILS_MASTER_KEY=secret
+SECRET_KEY_BASE=secret
 URL_HOST=mampf.mathi.uni-heidelberg.de
 URL_HOST_SHORT=http://mampf.media
 

lib/tasks/db.rake

@@ -33,7 +33,7 @@
 namespace :db do
   desc "Dumps the database to backups"
   task dump: :environment do
-    dump_fmt   = ensure_format(ENV.fetch("format", nil))
+    dump_fmt   = ensure_format(ENV.fetch("format"))
     dump_sfx   = suffix_for_format(dump_fmt)
     backup_dir = backup_directory(Rails.env, create: true)
     full_path  = nil
@@ -56,10 +56,10 @@ namespace :db do
   namespace :dump do
     desc "Dumps a specific table to backups"
     task table: :environment do
-      table_name = ENV.fetch("table", nil)
+      table_name = ENV.fetch("table")
 
       if table_name.present?
-        dump_fmt   = ensure_format(ENV.fetch("format", nil))
+        dump_fmt   = ensure_format(ENV.fetch("format"))
         dump_sfx   = suffix_for_format(dump_fmt)
         backup_dir = backup_directory(Rails.env, create: true)
         full_path  = nil
@@ -92,7 +92,7 @@ namespace :db do
 
   desc "Restores the database from a backup using PATTERN"
   task restore: :environment do
-    pattern = ENV.fetch("pattern", nil)
+    pattern = ENV.fetch("pattern")
 
     if pattern.present?
       file = nil