diff --git a/Gemfile.lock b/Gemfile.lock index f89ba70a..918b61b7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,14 +1,14 @@ PATH remote: . specs: - qless (0.11.0) - redis (>= 2.2, < 4.0.0.rc1) + qless (0.12.0) + redis (>= 2.2, < 5) rusage (~> 0.2.0) sentry-raven (~> 0.15.6) sinatra (>= 1.3, < 2.1) statsd-ruby (~> 1.3) thin (~> 1.6) - thor (~> 0.19.1) + thor (~> 0.19) vegas (~> 0.1.11) GEM @@ -43,18 +43,34 @@ GEM debugger-ruby_core_source (1.2.3) diff-lcs (1.2.4) eventmachine (1.0.9.1) - faraday (0.13.1) + faraday (1.7.0) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0.1) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.1) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) multipart-post (>= 1.2, < 3) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-net_http (1.0.1) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) faye-websocket (0.4.7) eventmachine (>= 0.12.0) ffi (1.9.18) - hitimes (1.2.6) http_parser.rb (0.5.3) method_source (0.8.2) mime-types (1.25) mini_portile (0.5.1) multi_json (1.12.2) - multipart-post (2.0.0) + multipart-post (2.1.1) nokogiri (1.6.0) mini_portile (~> 0.5.0) parallel (1.4.1) @@ -78,14 +94,14 @@ GEM binding_of_caller (>= 0.7) pry (>= 0.9.11) rack (1.6.4) - rack-protection (1.5.3) + rack-protection (1.5.5) rack rack-test (0.6.2) rack (>= 1.0) rainbow (2.2.2) rake rake (10.1.0) - redis (3.3.5) + redis (4.4.0) rspec (2.14.1) rspec-core (~> 2.14.0) rspec-expectations (~> 2.14.0) @@ -103,6 +119,7 @@ GEM ruby-progressbar (~> 1.7) unicode-display_width (~> 1.0, >= 1.0.1) ruby-progressbar (1.9.0) + ruby2_keywords (0.0.5) rubyzip (1.2.1) rusage (0.2.0) selenium-webdriver (2.53.4) @@ -120,13 +137,13 @@ GEM rack-protection (~> 1.4) tilt (>= 1.3, < 3) slop (3.6.0) - statsd-ruby (1.4.0) - thin (1.6.4) + statsd-ruby (1.5.0) + thin (1.5.4) daemons (~> 1.0, >= 1.0.9) eventmachine (~> 1.0, >= 1.0.4) rack (~> 1.0) - thor (0.19.4) - tilt (2.0.8) + thor (0.20.3) + tilt (2.0.10) timecop (0.7.1) unicode-display_width (1.3.0) vegas (0.1.11) @@ -156,4 +173,4 @@ DEPENDENCIES timecop BUNDLED WITH - 1.16.0 + 1.17.3 diff --git a/lib/qless.rb b/lib/qless.rb index ff1aa255..2a72b30f 100644 --- a/lib/qless.rb +++ b/lib/qless.rb @@ -27,6 +27,15 @@ module Qless require 'qless/lua_script' require 'qless/failure_formatter' +# monkey patch redis class to support gem version 3 and 4 +class Redis + + # in redis 4.0.0 the client command was introduced which overrides access to @client + def _client + @client + end unless method_defined?(:_client) +end + # The top level container for all things qless module Qless UnsupportedRedisVersionError = Class.new(Error) @@ -175,7 +184,7 @@ class Client def initialize(options = {}) # This is the redis instance we're connected to. Use connect so REDIS_URL # will be honored - @redis = options[:redis] || Redis.connect(options) + @redis = options[:redis] || Redis.new(options) @options = options assert_minimum_redis_version('2.5.5') @config = Config.new(self) @@ -195,7 +204,7 @@ def events # Events needs its own redis instance of the same configuration, because # once it's subscribed, we can only use pub-sub-like commands. This way, # we still have access to the client in the normal case - @events ||= ClientEvents.new(Redis.connect(@options)) + @events ||= ClientEvents.new(Redis.new(@options)) end def call(command, *argv) diff --git a/lib/qless/job.rb b/lib/qless/job.rb index 38831c5f..0437bbbc 100644 --- a/lib/qless/job.rb +++ b/lib/qless/job.rb @@ -172,7 +172,7 @@ def ttl end def reconnect_to_redis - @client.redis.client.reconnect + @client.redis._client.reconnect end def history diff --git a/lib/qless/middleware/redis_reconnect.rb b/lib/qless/middleware/redis_reconnect.rb index b0324e33..6fb62df3 100644 --- a/lib/qless/middleware/redis_reconnect.rb +++ b/lib/qless/middleware/redis_reconnect.rb @@ -15,7 +15,7 @@ def self.new(*redis_connections, &block) define_method :around_perform do |job| Array(block.call(job)).each do |redis| - redis.client.reconnect + redis._client.reconnect end super(job) diff --git a/lib/qless/worker/base.rb b/lib/qless/worker/base.rb index c91e72bd..dd154382 100644 --- a/lib/qless/worker/base.rb +++ b/lib/qless/worker/base.rb @@ -244,7 +244,7 @@ def current_job=(job) end def reconnect_each_client - uniq_clients.each { |client| client.redis.client.reconnect } + uniq_clients.each { |client| client.redis._client.reconnect } end end end diff --git a/qless.gemspec b/qless.gemspec index 797edee7..f8adb164 100644 --- a/qless.gemspec +++ b/qless.gemspec @@ -8,7 +8,7 @@ Gem::Specification.new do |s| s.licenses = ['MIT'] s.authors = ['Dan Lecocq', 'Myron Marston'] s.email = ['dan@moz.com', 'myron@moz.com'] - s.homepage = 'http://github.com/seomoz/qless' + s.homepage = 'http://github.com/seomoz/qless:HSC/DP-9467' s.summary = %q{A Redis-based queueing system} s.description = %q{ `qless` is meant to be a performant alternative to other queueing @@ -36,13 +36,13 @@ language-specific extension will also remain up to date. s.test_files = s.files.grep(/^(test|spec|features)\//) s.require_paths = ['lib'] - s.add_dependency 'redis', ['>= 2.2', '< 4.0.0.rc1'] + s.add_dependency 'redis', ['>= 2.2', '< 5'] s.add_dependency 'rusage', '~> 0.2.0' s.add_dependency 'sentry-raven', '~> 0.15.6' s.add_dependency 'sinatra', ['>= 1.3', '< 2.1'] s.add_dependency 'statsd-ruby', '~> 1.3' s.add_dependency 'thin', '~> 1.6' - s.add_dependency 'thor', '~> 0.19.1' + s.add_dependency 'thor', '~> 0.19' s.add_dependency 'vegas', '~> 0.1.11' end diff --git a/spec/integration/workers/forking_spec.rb b/spec/integration/workers/forking_spec.rb index 6b49d4c3..39feb073 100644 --- a/spec/integration/workers/forking_spec.rb +++ b/spec/integration/workers/forking_spec.rb @@ -52,7 +52,7 @@ module Qless # A job that just puts a word in a redis list to show that its done job_class = Class.new do def self.perform(job) - Redis.connect(url: job['redis']).rpush(job['key'], job['word']) + Redis.new(url: job['redis']).rpush(job['key'], job['word']) end end stub_const('JobClass', job_class) @@ -60,7 +60,7 @@ def self.perform(job) # Make jobs for each word words = %w{foo bar howdy} words.each do |word| - queue.put('JobClass', { redis: redis.client.id, key: key, word: word }) + queue.put('JobClass', { redis: redis._client.id, key: key, word: word }) end # Wait for the job to complete, and then kill the child process @@ -98,14 +98,14 @@ def self.perform(job) job.retry Process.kill(9, Process.pid) else - Redis.connect(url: job['redis']).rpush(job['key'], job['word']) + Redis.new(url: job['redis']).rpush(job['key'], job['word']) end end end stub_const('JobClass', job_class) # Put a job and run it, making sure it finally succeeds - queue.put('JobClass', { redis: redis.client.id, key: key, word: :foo }, + queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }, retries: 5) run_worker_concurrently_with(worker) do client.redis.brpop(key, timeout: 1).should eq([key.to_s, 'foo']) @@ -116,7 +116,7 @@ def self.perform(job) # Our mixin module sends a message to a channel mixin = Module.new do define_method :around_perform do |job| - Redis.connect(url: job['redis']).rpush(job['key'], job['word']) + Redis.new(url: job['redis']).rpush(job['key'], job['word']) super(job) end end @@ -129,7 +129,7 @@ def self.perform(job); end stub_const('JobClass', job_class) # Put a job in and run it - queue.put('JobClass', { redis: redis.client.id, key: key, word: :foo }) + queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }) run_worker_concurrently_with(worker) do client.redis.brpop(key, timeout: 1).should eq([key.to_s, 'foo']) end @@ -139,14 +139,14 @@ def self.perform(job); end # Our job class does nothing job_class = Class.new do def self.perform(job) - Redis.connect(url: job['redis']).rpush(job['key'], 'job') + Redis.new(url: job['redis']).rpush(job['key'], 'job') end end stub_const('JobClass', job_class) # Make jobs for each word 3.times do - queue.put('JobClass', { redis: redis.client.id, key: key }) + queue.put('JobClass', { redis: redis._client.id, key: key }) end # mixin module sends a message to a channel @@ -154,7 +154,7 @@ def self.perform(job) key = self.key mixin = Module.new do define_method :after_fork do - Redis.connect(url: redis_url).rpush(key, 'after_fork') + Redis.new(url: redis_url).rpush(key, 'after_fork') super() end end diff --git a/spec/integration/workers/serial_spec.rb b/spec/integration/workers/serial_spec.rb index 220bcb57..fd7bacc5 100644 --- a/spec/integration/workers/serial_spec.rb +++ b/spec/integration/workers/serial_spec.rb @@ -29,13 +29,13 @@ module Qless it 'does not leak threads' do job_class = Class.new do def self.perform(job) - Redis.connect(url: job['redis']).rpush(job['key'], job['word']) + Redis.new(url: job['redis']).rpush(job['key'], job['word']) end end stub_const('JobClass', job_class) # Put in a single job - queue.put('JobClass', { redis: redis.client.id, key: key, word: 'hello' }) + queue.put('JobClass', { redis: redis._client.id, key: key, word: 'hello' }) expect do run_jobs(worker, 1) do expect(redis.brpop(key, timeout: 1)).to eq([key.to_s, 'hello']) @@ -47,7 +47,7 @@ def self.perform(job) # A job that just puts a word in a redis list to show that its done job_class = Class.new do def self.perform(job) - Redis.connect(url: job['redis']).rpush(job['key'], job['word']) + Redis.new(url: job['redis']).rpush(job['key'], job['word']) end end stub_const('JobClass', job_class) @@ -55,7 +55,7 @@ def self.perform(job) # Make jobs for each word words = %w{foo bar howdy} words.each do |word| - queue.put('JobClass', { redis: redis.client.id, key: key, word: word }) + queue.put('JobClass', { redis: redis._client.id, key: key, word: word }) end # Wait for the job to complete, and then kill the child process @@ -94,7 +94,7 @@ def self.perform(job) # Job that sleeps for a while on the first pass job_class = Class.new do def self.perform(job) - redis = Redis.connect(url: job['redis']) + redis = Redis.new(url: job['redis']) if redis.get(job['jid']).nil? redis.set(job['jid'], '1') redis.rpush(job['key'], job['word']) @@ -110,7 +110,7 @@ def self.perform(job) # Put this job into the queue and then busy-wait for the job to be # running, time it out, then make sure it eventually completes - queue.put('JobClass', { redis: redis.client.id, key: key, word: :foo }, + queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }, jid: 'jid') run_jobs(worker, 2) do expect(redis.brpop(key, timeout: 1)).to eq([key.to_s, 'foo']) @@ -171,7 +171,7 @@ def self.perform(job) # A class that sends a message and sleeps for a bit job_class = Class.new do def self.perform(job) - redis = Redis.connect(url: job['redis']) + redis = Redis.new(url: job['redis']) redis.rpush(job['key'], job['word']) redis.brpop(job['key']) end @@ -179,9 +179,9 @@ def self.perform(job) stub_const('JobClass', job_class) # Put this job into the queue and then have the worker lose its lock - queue.put('JobClass', { redis: redis.client.id, key: key, word: :foo }, + queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }, priority: 10, jid: 'jid') - queue.put('JobClass', { redis: redis.client.id, key: key, word: :foo }, + queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }, priority: 5, jid: 'other') run_jobs(worker, 1) do @@ -213,14 +213,14 @@ def self.perform(job) retry_on Kaboom def self.perform(job) - Redis.connect(url: job['redis']).rpush(job['key'], job['word']) + Redis.new(url: job['redis']).rpush(job['key'], job['word']) raise Kaboom end end stub_const('JobClass', job_class) # Put a job and run it, making sure it gets retried - queue.put('JobClass', { redis: redis.client.id, key: key, word: :foo }, + queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }, jid: 'jid', retries: 10) run_jobs(worker, 1) do redis.brpop(key, timeout: 1).should eq([key.to_s, 'foo']) diff --git a/spec/unit/qless_spec.rb b/spec/unit/qless_spec.rb index 7e41af58..0abf3723 100644 --- a/spec/unit/qless_spec.rb +++ b/spec/unit/qless_spec.rb @@ -22,7 +22,7 @@ def redis_double(overrides = {}) before do redis.stub(:script) # so no scripts get loaded - redis_class.stub(connect: redis) + redis_class.stub(new: redis) end describe '#worker_name' do @@ -58,7 +58,7 @@ def redis_double(overrides = {}) end it 'allows the redis connection to be passed directly in' do - redis_class.should_not_receive(:connect) + redis_class.should_not_receive(:new) client = Qless::Client.new(redis: redis) client.redis.should be(redis)