perf(storage) replace redis keys call with scan (#106)

fffonion · Sep 5, 2023 · d806c19 · d806c19
1 parent 78eb7bf
commit d806c19
Show file tree

Hide file tree

Showing 2 changed files with 148 additions and 5 deletions.
diff --git a/lib/resty/acme/storage/redis.lua b/lib/resty/acme/storage/redis.lua
@@ -3,6 +3,7 @@ local util = require "resty.acme.util"
 local fmt   = string.format
 local log   = util.log
 local ngx_ERR = ngx.ERR
+local unpack = unpack
 
 local _M = {}
 local mt = {__index = _M}
@@ -20,6 +21,7 @@ function _M.new(conf)
       ssl_verify = conf.ssl_verify or false,
       ssl_server_name = conf.ssl_server_name,
       namespace = conf.namespace or "",
+      scan_count = conf.scan_count or 10,
     },
     mt
   )
@@ -133,11 +135,28 @@ local empty_table = {}
 function _M:list(prefix)
   prefix = prefix or ""
   prefix = self.namespace .. prefix
-  local res, err = op(self, 'keys', prefix .. "*")
-  if not res or res == ngx.null then
-    return empty_table, err
-  end
-  return remove_namespace(self.namespace, res), err
+
+  local cursor = "0"
+  local data = {}
+  local res, err
+
+  repeat
+    res, err = op(self, 'scan', cursor, 'match', prefix .. "*", 'count', self.scan_count)
+
+    if not res or res == ngx.null then
+      return empty_table, err
+    end
+
+    local keys
+    cursor, keys = unpack(res)
+
+    for i=1,#keys do
+      data[#data+1] = keys[i]
+    end
+
+  until cursor == "0"
+
+  return remove_namespace(self.namespace, data), err
 end
 
 return _M
diff --git a/t/storage/redis.t b/t/storage/redis.t
@@ -431,4 +431,128 @@ nil
 --- no_error_log
 [error]
 
+=== TEST 14: Redis list keys with multiple scan calls
+--- http_config eval: $::HttpConfig
+--- config
+    location =/t {
+        content_by_lua_block {
+            local st = test_lib.new(test_cfg)
+            for i=1,50 do
+                local err = st:set(string.format("test14:%02d", i), string.format("value%02d", i))
+                ngx.say(err)
+            end
+
+            local keys, err = st:list("test14")
+            ngx.say(err)
+            table.sort(keys)
+            for _, p in ipairs(keys) do ngx.say(p) end
+        }
+    }
+--- request
+    GET /t
+--- response_body_like eval
+"nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+nil
+test14:01
+test14:02
+test14:03
+test14:04
+test14:05
+test14:06
+test14:07
+test14:08
+test14:09
+test14:10
+test14:11
+test14:12
+test14:13
+test14:14
+test14:15
+test14:16
+test14:17
+test14:18
+test14:19
+test14:20
+test14:21
+test14:22
+test14:23
+test14:24
+test14:25
+test14:26
+test14:27
+test14:28
+test14:29
+test14:30
+test14:31
+test14:32
+test14:33
+test14:34
+test14:35
+test14:36
+test14:37
+test14:38
+test14:39
+test14:40
+test14:41
+test14:42
+test14:43
+test14:44
+test14:45
+test14:46
+test14:47
+test14:48
+test14:49
+test14:50
+"
+--- no_error_log
+[error]