Sidekiq: adds a script to terminate orphan workers

cookbooks/sidekiq/attributes/default.rb

@@ -9,13 +9,13 @@
   # only be installed on to a utility instance that matches
   # the name
   :utility_name => 'sidekiq',
-  
+
   # Number of workers (not threads)
   :workers => 1,
-  
+
   # Concurrency
   :concurrency => 25,
-  
+
   # Queues
   :queues => {
     # :queue_name => priority
@@ -30,5 +30,17 @@
 
   # Timeout (in seconds) to use when terminating a bloated process
   # this is passed as a parameter to sidekiqctl, invoked inside /engineyard/bin/sidekiq
-  :timeout => 115
+  :timeout => 115,
+
+  # Setting this to true installs a cron job that
+  # regularly terminates sidekiq workers that aren't being monitored by monit,
+  # and terminates those workers
+  #
+  # default: false
+  :orphan_monitor_enabled => false,
+
+  # sidekiq_orphan_monitor cron schedule
+  #
+  # default: every 5 minutes
+  :orphan_monitor_cron_schedule => "*/5 * * * *"
 }

cookbooks/sidekiq/files/default/sidekiq_orphan_monitor

@@ -0,0 +1,30 @@
+containsElement () {
+  local e match="$1"
+  shift
+  for e; do [[ "$e" == "$match" ]] && return 0; done
+  return 1
+}
+
+app_name=$1
+sidekiq_pids=`cat /var/run/engineyard/sidekiq/$app_name/sidekiq_*.pid`
+
+sidekiq_parent_pids=()
+for sidekiq_pid in $sidekiq_pids; do
+  parent_pid="$(ps -o ppid= $sidekiq_pid | tr -d '[:space:]')"
+  sidekiq_parent_pids+="$parent_pid "
+done
+
+running_sidekiqs="$(ps -ef | grep [s]idekiq | awk '{print $2}')"
+echo "Running sidekiqs: $running_sidekiqs"
+echo "Monit PIDs: $sidekiq_pids"
+echo "Parent PIDs: $sidekiq_parent_pids"
+for sidekiq in $running_sidekiqs; do
+  if (containsElement $sidekiq $sidekiq_pids); then
+    echo "Not terminating $sidekiq - it is being monitored by monit"
+  elif (containsElement $sidekiq $sidekiq_parent_pids); then
+    echo "Not terminating $sidekiq - it is a parent of a monit sidekiq process"
+  else
+    echo "Terminating $sidekiq"
+    sudo kill $sidekiq
+  fi
+done

cookbooks/sidekiq/recipes/setup.rb

@@ -54,5 +54,28 @@
         variables(node[:sidekiq])
       end
     end
+
+    if node[:sidekiq][:orphan_monitor_enabled]
+      cookbook_file '/engineyard/bin/sidekiq_orphan_monitor' do
+        source 'sidekiq_orphan_monitor'
+        owner node[:owner_name]
+        group node[:owner_name]
+        mode 0755
+        backup false
+        action :create
+      end
+
+      cron 'sidekiq_orphan_monitor' do
+        user    node[:owner_name]
+        action  :create
+        minute  node[:sidekiq][:orphan_monitor_cron_schedule].split[0]
+        hour    node[:sidekiq][:orphan_monitor_cron_schedule].split[1]
+        day     node[:sidekiq][:orphan_monitor_cron_schedule].split[2]
+        month   node[:sidekiq][:orphan_monitor_cron_schedule].split[3]
+        weekday node[:sidekiq][:orphan_monitor_cron_schedule].split[4]
+        command "/engineyard/bin/sidekiq_orphan_monitor #{app_name}"
+      end
+    end
+
   end
 end