diff --git a/redis.submodule/.github/workflows/ci.yml b/redis.submodule/.github/workflows/ci.yml new file mode 100644 index 0000000..439e3f3 --- /dev/null +++ b/redis.submodule/.github/workflows/ci.yml @@ -0,0 +1,49 @@ +name: CI + +on: [push, pull_request] + +jobs: + + test-ubuntu-latest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: make + run: make + - name: test + run: | + sudo apt-get install tcl8.5 + ./runtest --verbose + - name: module api test + run: ./runtest-moduleapi --verbose + + build-ubuntu-old: + runs-on: ubuntu-16.04 + steps: + - uses: actions/checkout@v1 + - name: make + run: make + + build-macos-latest: + runs-on: macos-latest + steps: + - uses: actions/checkout@v1 + - name: make + run: make + + biuld-32bit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: make + run: | + sudo apt-get update && sudo apt-get install libc6-dev-i386 + make 32bit + + build-libc-malloc: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: make + run: make MALLOC=libc + diff --git a/redis.submodule/.github/workflows/daily.yml b/redis.submodule/.github/workflows/daily.yml new file mode 100644 index 0000000..acc4dd3 --- /dev/null +++ b/redis.submodule/.github/workflows/daily.yml @@ -0,0 +1,81 @@ +name: Daily + +on: + schedule: + - cron: '0 7 * * *' + +jobs: + + test-jemalloc: + runs-on: ubuntu-latest + timeout-minutes: 1200 + steps: + - uses: actions/checkout@v1 + - name: make + run: make + - name: test + run: | + sudo apt-get install tcl8.5 + ./runtest --accurate --verbose + - name: module api test + run: ./runtest-moduleapi --verbose + + test-libc-malloc: + runs-on: ubuntu-latest + timeout-minutes: 1200 + steps: + - uses: actions/checkout@v1 + - name: make + run: make MALLOC=libc + - name: test + run: | + sudo apt-get install tcl8.5 + ./runtest --accurate --verbose + - name: module api test + run: ./runtest-moduleapi --verbose + + test-32bit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: make + run: | + sudo apt-get update && sudo apt-get install libc6-dev-i386 + make 32bit + - name: test + run: | + sudo apt-get install tcl8.5 + ./runtest --accurate --verbose + - name: module api test + run: | + make -C tests/modules 32bit # the script below doesn't have an argument, we must build manually ahead of time + ./runtest-moduleapi --verbose + + test-tls: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: make + run: | + make BUILD_TLS=yes + - name: test + run: | + sudo apt-get install tcl8.5 tcl-tls + ./utils/gen-test-certs.sh + ./runtest --accurate --verbose --tls + - name: module api test + run: ./runtest-moduleapi --verbose --tls + + test-valgrind: + runs-on: ubuntu-latest + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v1 + - name: make + run: make valgrind + - name: test + run: | + sudo apt-get install tcl8.5 valgrind -y + ./runtest --valgrind --verbose --clients 1 + - name: module api test + run: ./runtest-moduleapi --valgrind --verbose --clients 1 diff --git a/redis.submodule/.gitignore b/redis.submodule/.gitignore index a188cfc..e445fd2 100644 --- a/redis.submodule/.gitignore +++ b/redis.submodule/.gitignore @@ -1,5 +1,8 @@ .*.swp *.o +*.xo +*.so +*.d *.log dump.rdb redis-benchmark @@ -28,3 +31,5 @@ deps/lua/src/liblua.a .prerequisites *.dSYM Makefile.dep +.vscode/* +.idea/* diff --git a/redis.submodule/00-RELEASENOTES b/redis.submodule/00-RELEASENOTES index 285ae84..c6ee442 100644 --- a/redis.submodule/00-RELEASENOTES +++ b/redis.submodule/00-RELEASENOTES @@ -1,4 +1,4 @@ -Redis 5.0 release notes +Redis 6.0 release notes ======================= -------------------------------------------------------------------------------- @@ -12,3162 +12,2126 @@ SECURITY: There are security fixes in the release. -------------------------------------------------------------------------------- ================================================================================ -Redis 5.0.6 Released Wed Sep 25 12:33:56 CEST 2019 +Redis 6.0.5 Released Tue Jun 09 11:56:08 CEST 2020 ================================================================================ -Upgrade urgency CRITICAL: Only in case of exposed instances to untrusted users. +Upgrade urgency MODERATE: several bugs with moderate impact are fixed here. -This Redis release, 5.0.6, is a bugfix and enhancement release. The most -important bugfix is a corruption related to the HyperLogLog. A malformed -HyperLogLog string could cause an invalid access to the memory. At a first -glance the vulnerability appears to be not exploitable but just a DoS. The -way to trigger the issue is complex, we'll not provide any information about -how to do that for the users safety. +The most important issues are listed here: -Other significant changes in this release: +* Fix handling of speical chars in ACL LOAD. +* Make Redis Cluster more robust about operation errors that may lead + to two clusters to mix together. +* Revert the sendfile() implementation of RDB transfer. It causes some delay. +* Fix TLS certificate loading for chained certificates. +* Fix AOF rewirting of KEEPTTL SET option. +* Fix MULTI/EXEC behavior during -BUSY script errors. -* New modules APIs merged from Redis unstable to Redis 5. -* Some memory optimization related to objects creation. -* Fixes to flushSlaveOutputBuffer() that make sure that SHUTDOWN will - transfer pending buffers to replicas. +And this is the full list of commits: -This is the full list of commits: - -antirez in commit 7a41047a: - RDB: fix MODULE_AUX loading by continuing to next opcode. - 1 file changed, 1 insertion(+) - -Oran Agra in commit 4eb3028b: - missing per-skiplist overheads in MEMORY USAGE - 1 file changed, 3 insertions(+), 1 deletion(-) +antirez in commit ee8dd01bb: + Temporary fix for #7353 issue about EVAL during -BUSY. + 1 file changed, 9 insertions(+) -Oran Agra in commit 5d09f9bc: - RM_Log - add support for logging without a context or context without module - 1 file changed, 6 insertions(+), 4 deletions(-) +xhe in commit a4a856d53: + return the correct proto version HELLO should return the current proto version, while the code hardcoded 3 + 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 2810de9f: - Cluster: abort loading nodes data if vars arguments are unbalanced. - 1 file changed, 1 insertion(+) +Oran Agra in commit e2046b300: + Don't queue commands in an already aborted MULTI state + 1 file changed, 7 insertions(+) -antirez in commit f5c63ce0: - More strict checks and better comments in flushSlaveOutputBuffers(). - 1 file changed, 18 insertions(+), 7 deletions(-) +Oran Agra in commit b35fdf1de: + Avoid rejecting WATCH / UNWATCH, like MULTI/EXEC/DISCARD + 1 file changed, 4 insertions(+), 2 deletions(-) -antirez in commit 7f289c3b: - Improve comment in flushSlavesOutputBuffers(). - 1 file changed, 2 insertions(+), 1 deletion(-) +zhaozhao.zz in commit 1d7bf208c: + AOF: append origin SET if no expire option + 2 files changed, 23 insertions(+), 8 deletions(-) -antirez in commit 7ab62d4b: - Replication: clarify why repl_put_online_on_ack exists at all. - 2 files changed, 34 insertions(+), 10 deletions(-) +Oran Agra in commit 676445ad9: + fix disconnectSlaves, to try to free each slave. + 1 file changed, 1 deletion(-) -zhaozhao.zz in commit 495dd0da: - networking: flushSlavesOutputBuffers bugfix - 1 file changed, 2 insertions(+), 4 deletions(-) +zhaozhao.zz in commit 4846c0c8a: + donot free protected client in freeClientsInAsyncFreeQueue + 1 file changed, 9 insertions(+), 3 deletions(-) -Salvatore Sanfilippo in commit c1ccf0f1: - Merge pull request #6366 from oranagra/5.0_rm_reply_cstring -Salvatore Sanfilippo in commit a50dad73: - Merge pull request #6365 from oranagra/5.0_module_aux -Oran Agra in commit d6294d05: - RM_ReplyWithCString was missing registration +Oran Agra in commit f33de403e: + fix pingoff test race 1 file changed, 1 insertion(+) -Oran Agra in commit 8c56fc86: - Fix to module aux data rdb format for backwards compatibility with old check-rdb - 1 file changed, 9 insertions(+), 1 deletion(-) - -Oran Agra in commit 98b1314f: - Implement module api for aux data in rdb - 9 files changed, 431 insertions(+), 18 deletions(-) - -antirez in commit 08b03e23: - redis-cli: always report server errors on read errors. - 1 file changed, 8 insertions(+), 1 deletion(-) - -wubostc in commit 239069de: - Reduce the calling stack - 1 file changed, 2 insertions(+), 3 deletions(-) - -antirez in commit 90bf6313: - Make EMBSTR case of #6261 more obvious. +Kevin Fwu in commit 49af4d07e: + Fix TLS certificate loading for chained certificates. 1 file changed, 1 insertion(+), 1 deletion(-) -chendianqiang in commit 2f8a0749: - make memory usage consistent of robj with OBJ_ENCODING_INT - 1 file changed, 9 insertions(+), 4 deletions(-) - -antirez in commit 436ed56d: - HyperLogLog: fix the fix of a corruption bug. - 1 file changed, 1 insertion(+), 2 deletions(-) - -John Sully in commit 680f89fb: - Fix HLL corruption bug - 1 file changed, 1 insertion(+) +antirez in commit 329fddbda: + Revert "Implements sendfile for redis." + 2 files changed, 2 insertions(+), 55 deletions(-) -swilly22 in commit 388efbf8: - Extend REDISMODULE_CTX_FLAGS to indicate if redis is currently loading from either RDB or AOF - 2 files changed, 5 insertions(+) +antirez in commit 925a2cd5a: + Revert "avoid using sendfile if tls-replication is enabled" + 1 file changed, 27 insertions(+), 34 deletions(-) -Itamar Haber in commit 0ccbdcee: - Uses addReplyBulkCString - 1 file changed, 1 insertion(+), 1 deletion(-) +Liu Zhen in commit 84a7a9058: + fix clusters mixing accidentally by gossip + 1 file changed, 10 insertions(+), 2 deletions(-) -Itamar Haber in commit 707a59c6: - Adds RedisModule_ReplyWithCString - 2 files changed, 13 insertions(+) +antirez in commit cd63359a1: + Fix handling of special chars in ACL LOAD. + 1 file changed, 8 insertions(+), 4 deletions(-) ================================================================================ -Redis 5.0.5 Released Wed May 15 17:57:41 CEST 2019 +Redis 6.0.4 Released Thu May 28 11:36:45 CEST 2020 ================================================================================ -Upgrade urgency CRITICAL: This release fixes an important AOF fysnc bug - and other less critical issues. +Upgrade urgency CRITICAL: this release fixes a severe replication bug. + +Redis 6.0.4 fixes a critical replication bug caused by a new feature introduced +in Redis 6. The feature, called "meaningful offset" and strongly wanted by +myself (antirez) was an improvement that avoided that masters were no longer +able, during a failover where they were demoted to replicas, to partially +synchronize with the new master. In short the feature was able to avoid full +synchronizations with RDB. How did it work? By trimming the replication backlog +of the final "PING" commands the master was sending in the replication channel: +this way the replication offset would no longer go "after" the one of the +promoted replica, allowing the master to just continue in the same replication +history, receiving only a small data difference. + +However after the introduction of the feature we (the Redis core team) quickly +understood there was something wrong: the apparently harmless feature had +many bugs, and the last bug we discovered, after a joined effort of multiple +people, we were not even able to fully understand after fixing it. Enough was +enough, we decided that the complexity cost of this feature was too high. +So Redis 6.0.4 removes the feature entirely, and fixes the data corruption that +it was able to cause. + +However there are two facts to take in mind. + +Fact 1: Setups using chained replication, that means that certain replicas +are replicating from other replicas, up to Redis 6.0.3 can experience data +corruption. For chained replication we mean that: + + +--------+ +---------+ +-------------+ + | master |--------->| replica |-------->| sub-replica | + +--------+ +---------+ +-------------+ + + +People using chained replication SHOULD UPGRADE ASAP away from Redis 6.0.0, +6.0.1, 6.0.2 or 6.0.3 to Redis 6.0.4. + +To be clear, people NOT using this setup, but having just replicas attached +directly to the master, SHOUDL NOT BE in danger of any problem. But we +are no longer confident on 6.0.x replication implementation complexities +so we suggest to upgrade to 6.0.4 to everybody using an older 6.0.3 release. +We just so far didn't find any bug that affects Redis 6.0.3 that does not +involve chained replication. + +People starting with Redis 6.0.4 are fine. People with Redis 5 are fine. +People upgrading from Redis 5 to Redis 6.0.4 are fine. +TLDR: The problem is with users of 6.0.0, 6.0.1, 6.0.2, 6.0.3. + +Fact 2: Upgrading from Redis 6.0.x to Redis 6.0.4, IF AND ONLY IF you +use chained replication, requires some extra care: + +1. Once you attach your new Redis 6.0.4 instance as a replica of the current + Redis 6.0.x master, you should wait for the first full synchronization, + then you should promote it right away, if your setup involves chained + replication. Don't give it the time to do a new partial synchronization + in the case the link between the master and the replica will break in + the mean time. + +2. As an additional care, you may want to set the replication ping period + to a very large value (for instance 1000000) using the following command: + + CONFIG SET repl-ping-replica-period 1000000 + + Note that if you do "1" with care, "2" is not needed. + However if you do it, make sure to later restore it to its default: + + CONFIG SET repl-ping-replica-period 10 + +So this is the main change in Redis 6. Later we'll find a different way in +order to achieve what we wanted to achieve with the Meaningful Offset feature, +but without the same complexity. + +Other changes in this release: + +* PSYNC2 tests improved. +* Fix a rare active defrag edge case bug leading to stagnation +* Fix Redis 6 asserting at startup in 32 bit systems. +* Redis 6 32 bit is now added back to our testing environments. +* Fix server crash for STRALGO command, +* Implement sendfile for RDB transfer. +* TLS fixes. +* Make replication more resistant by disconnecting the master if we + detect a protocol error. Basically we no longer accept inline protocol + from the master. +* Other improvements in the tests. +Regards, +antirez -Dear user, - -Redis 5.0.5 fixes an important issue with AOF and adds multiple very useful -modules APIs. Moreover smaller bugs in other parts of Redis are fixed in -this release. +This is the full list of commits: -The AOF bug ------------ +antirez in commit 59cd4c9f6: + Test: take PSYNC2 test master timeout high during switch. + 1 file changed, 1 deletion(-) -The AOF bug happens when the fsync policy is set to "everysec", which is the -default: if the write load in the server drops immediately, the commands -executed in the latest second may not be fsync-ed to disk as it should. -This may lead to data loss in case the write load drops immediately and -successively a server crash happens. +antirez in commit 6c1bb7b19: + Test: add the tracking unit as default. + 1 file changed, 1 insertion(+) -Other things in this release ----------------------------- +Oran Agra in commit 1aee695e5: + tests: find_available_port start search from next port + 1 file changed, 12 insertions(+), 7 deletions(-) -* Streams: a bug in the iterator could prevent certain items to be returned in - range queries under specific conditions. -* Memleak in bitfieldCommand fixed. -* Modules API: Preserve client->id for blocked clients. -* Fix memory leak when rewriting config file in case of write errors. -* New modules API: RedisModule_GetKeyNameFromIO(). -* Fix non critical bugs in diskless replication. -* New mdouels API: command filtering. See RedisModule_RegisterCommandFilter(); -* Tests improved to be more deterministic. -* Fix a Redis Cluster bug, manual failover may abort because of the master - sending PINGs to the replicas. +Oran Agra in commit a2ae46352: + tests: each test client work on a distinct port range + 5 files changed, 39 insertions(+), 27 deletions(-) -The following is the full list of commmits. +Oran Agra in commit 86e562d69: + 32bit CI needs to build modules correctly + 2 files changed, 7 insertions(+), 2 deletions(-) -Regards, -Salvatore +Oran Agra in commit ab2984b1e: + adjust revived meaningful offset tests + 1 file changed, 39 insertions(+), 20 deletions(-) -Christian Zeller in commit 1cac9b4b: - Typo fixes in CONTRIBUTING - 1 file changed, 2 insertions(+), 2 deletions(-) +Oran Agra in commit 1ff5a222d: + revive meaningful offset tests + 2 files changed, 213 insertions(+) -antirez in commit f63c5c7b: - Update CONTRIBUTING with present info. - 1 file changed, 15 insertions(+), 5 deletions(-) +antirez in commit cc549b46a: + Replication: showLatestBacklog() refactored out. + 3 files changed, 36 insertions(+), 25 deletions(-) -antirez in commit 668661da: - Narrow the effects of PR #6029 to the exact state. - 1 file changed, 17 insertions(+), 5 deletions(-) +antirez in commit 377dd0515: + Drop useless line from replicationCacheMaster(). + 1 file changed, 2 deletions(-) -chendianqiang in commit 3c2800e3: - stop ping when client pause - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit 3f8d113f1: + Another meaningful offset test removed. + 1 file changed, 100 deletions(-) -antirez in commit 7ac7ffd5: - Test: fix slowlog test false positive. - 1 file changed, 3 insertions(+), 1 deletion(-) +antirez in commit d4541349d: + Remove the PSYNC2 meaningful offset test. + 2 files changed, 113 deletions(-) -antirez in commit cc101721: - Make comment in getClientOutputBufferMemoryUsage() describing the present. - 1 file changed, 1 insertion(+), 8 deletions(-) +antirez in commit 2112a5702: + Remove the meaningful offset feature. + 4 files changed, 10 insertions(+), 93 deletions(-) -WuYunlong in commit 72420b0d: - Do not active expire keys in the background when the switch is off. - 1 file changed, 6 insertions(+), 4 deletions(-) +antirez in commit d2eb6e0b4: + Set a protocol error if master use the inline protocol. + 1 file changed, 17 insertions(+), 2 deletions(-) -liaotonglang in commit 33a50d24: - delete sdsTest() from REDIS_TEST - 1 file changed, 2 deletions(-) +Oran Agra in commit 9c1df3b76: + daily CI test with tls + 1 file changed, 15 insertions(+) -zhaozhao.zz in commit 6a92836f: - test cases: skiptill -> skip-till - 1 file changed, 1 insertion(+), 1 deletion(-) +Oran Agra in commit 115ed1911: + avoid using sendfile if tls-replication is enabled + 1 file changed, 34 insertions(+), 27 deletions(-) -Oran Agra in commit f179f71e: - make replication tests more stable on slow machines - 3 files changed, 34 insertions(+), 4 deletions(-) +antirez in commit 11c748aac: + Replication: log backlog creation event. + 1 file changed, 3 insertions(+) -Yossi Gottlieb in commit 1825a4ec: - Add runtest-moduleapi with commandfilter coverage. - 5 files changed, 63 insertions(+), 28 deletions(-) +antirez in commit 8f1013722: + Test: PSYNC2 test can now show server logs. + 1 file changed, 88 insertions(+), 25 deletions(-) -Yossi Gottlieb in commit 9d20fdb4: - fix: missing initialization. - 3 files changed, 1 insertion(+) +antirez in commit 2e591fc4a: + Clarify what is happening in PR #7320. + 1 file changed, 5 insertions(+), 1 deletion(-) -antirez in commit ded1980e: - Test: disable module testing for now. - 1 file changed, 1 deletion(-) +zhaozhao.zz in commit cbb51fb8f: + PSYNC2: second_replid_offset should be real meaningful offset + 1 file changed, 3 insertions(+), 3 deletions(-) -Yossi Gottlieb in commit c3df78c2: - CommandFilter API: REDISMODULE_CMDFILTER_NOSELF. - 4 files changed, 62 insertions(+), 15 deletions(-) +Oran Agra in commit e0fc88b4d: + add CI for 32bit build + 2 files changed, 34 insertions(+) -Yossi Gottlieb in commit 8d38ef20: - CommandFilter API: fix UnregisterCommandFilter. - 1 file changed, 2 insertions(+), 3 deletions(-) +antirez in commit e3f864b5f: + Make disconnectSlaves() synchronous in the base case. + 3 files changed, 20 insertions(+), 9 deletions(-) -Yossi Gottlieb in commit 9b7009b1: - CommandFilter API: Add unregister option. - 4 files changed, 126 insertions(+), 32 deletions(-) +ShooterIT in commit 8af1e513f: + Implements sendfile for redis. + 2 files changed, 55 insertions(+), 2 deletions(-) -Yossi Gottlieb in commit 05802442: - CommandFilter API: Extend documentation. - 1 file changed, 43 insertions(+), 5 deletions(-) +antirez in commit 3c21418cd: + Fix #7306 less aggressively. + 2 files changed, 29 insertions(+), 17 deletions(-) -Yossi Gottlieb in commit d5194daf: - CommandFilter API: hellofilter and tests. - 2 files changed, 47 insertions(+), 5 deletions(-) +Madelyn Olson in commit e201f83ce: + EAGAIN for tls during diskless load + 1 file changed, 4 insertions(+) -Yossi Gottlieb in commit 8897c154: - CommandFilter API: Support Lua and RM_call() flows. - 2 files changed, 18 insertions(+), 7 deletions(-) +Qu Chen in commit 58fc456cb: + Disconnect chained replicas when the replica performs PSYNC with the master always to avoid replication offset mismatch between master and chained replicas. + 2 files changed, 60 insertions(+), 3 deletions(-) -Yossi Gottlieb in commit 6dd5bad4: - CommandFilter API: More cleanup. - 2 files changed, 10 insertions(+), 29 deletions(-) +hwware in commit 3febc5c29: + using moreargs variable + 1 file changed, 2 insertions(+), 2 deletions(-) -Yossi Gottlieb in commit 83026101: - Add command filter Module API tests. - 2 files changed, 28 insertions(+) +hwware in commit 8d6738559: + fix server crash for STRALGO command + 1 file changed, 2 insertions(+), 2 deletions(-) -Yossi Gottlieb in commit dc5edc7b: - Add command filtering argument handling API. - 3 files changed, 132 insertions(+), 13 deletions(-) +ShooterIT in commit 7a35eec54: + Replace addDeferredMultiBulkLength with addReplyDeferredLen in comment + 1 file changed, 2 insertions(+), 2 deletions(-) -Yossi Gottlieb in commit 5f29e2e2: - Initial command filter experiment. - 6 files changed, 161 insertions(+), 2 deletions(-) +Yossi Gottlieb in commit f93e1417b: + TLS: Improve tls-protocols clarity in redis.conf. + 1 file changed, 3 insertions(+), 2 deletions(-) -Oran Agra in commit e1839ab3: - diskless fork kept streaming RDB to a disconnected slave - 1 file changed, 10 insertions(+) +ShooterIT in commit d0c9e4454: + Fix reply bytes calculation error + 1 file changed, 1 insertion(+), 1 deletion(-) -Oran Agra in commit 3b207b89: - diskless replication - notify slave when rdb transfer failed +zhaozhao.zz in commit 1cde6a060: + Tracking: flag CLIENT_TRACKING_BROKEN_REDIR when redir broken 1 file changed, 1 insertion(+) -antirez in commit 7e350b09: - More sensible name for function: restartAOFAfterSYNC(). - 1 file changed, 7 insertions(+), 3 deletions(-) +Oran Agra in commit 436be3498: + fix a rare active defrag edge case bug leading to stagnation + 4 files changed, 146 insertions(+), 23 deletions(-) -antirez in commit 91238a60: - Mostly aesthetic changes to restartAOF(). - 1 file changed, 7 insertions(+), 3 deletions(-) +Oran Agra in commit f9d2ffdc5: + improve DEBUG MALLCTL to be able to write to write only fields. + 1 file changed, 27 insertions(+), 7 deletions(-) -oranagra in commit ee2da67c: - bugfix to restartAOF, exit will never happen since retry will get negative. +hujie in commit d7968ee92: + fix clear USER_FLAG_ALLCOMMANDS flag in acl 1 file changed, 5 insertions(+), 4 deletions(-) -Oran Agra in commit 78022492: - Add log when server dies of SIGTERM during loading - 1 file changed, 1 insertion(+) +ShooterIT in commit a902e6b25: + Redis Benchmark: generate random test data + 1 file changed, 12 insertions(+), 1 deletion(-) -Yossi Gottlieb in commit 232dca7f: - Add RedisModule_GetKeyNameFromIO(). - 8 files changed, 30 insertions(+), 17 deletions(-) +hwware in commit 9564ed7c3: + Redis-Benchmark: avoid potentical memmory leaking + 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 7f98129a: - MANIFESTO: simplicity and lock-in. - 1 file changed, 5 insertions(+), 1 deletion(-) +WuYunlong in commit 2e4182743: + Handle keys with hash tag when computing hash slot using tcl cluster client. + 1 file changed, 23 insertions(+), 2 deletions(-) -antirez in commit 71265fe3: - MANIFESTO v2. - 1 file changed, 41 insertions(+), 6 deletions(-) +WuYunlong in commit eb2c8b2c6: + Add a test to prove current tcl cluster client can not handle keys with hash tag. + 1 file changed, 7 insertions(+), 1 deletion(-) -yongman in commit 8115be6e: - Fix uint64_t hash value in active defrag - 1 file changed, 3 insertions(+), 3 deletions(-) +ShooterIT in commit 928e6976b: + Use dictSize to get the size of dict in dict.c + 1 file changed, 2 insertions(+), 2 deletions(-) -Angus Pearson in commit 90e7b5a9: - Enlarge error buffer in redis-check-aof.c to remove compiler warning of output truncation through snprintf format string - 1 file changed, 1 insertion(+), 1 deletion(-) +Madelyn Olson in commit cdcf5af5a: + Converge hash validation for adding and removing + 1 file changed, 21 insertions(+), 14 deletions(-) -zhaozhao.zz in commit 43151baf: - fix memory leak when rewrite config file - 1 file changed, 3 insertions(+), 4 deletions(-) +Benjamin Sergeant in commit e8b09d220: + do not handle --cluster-yes for cluster fix mode + 1 file changed, 16 insertions(+), 7 deletions(-) -唐权 in commit d3c17c9d: - Update ziplist.c +Benjamin Sergeant in commit 57b4fb0d8: + fix typo ... 1 file changed, 1 insertion(+), 1 deletion(-) -stan011 in commit 296bd097: - change the comments there may have a mis type +Benjamin Sergeant in commit 29f25e411: + Redis-cli 6.0.1 `--cluster-yes` doesn't work (fix #7246) + 1 file changed, 5 insertions(+), 1 deletion(-) + +Oran Agra in commit 00d8b92b8: + fix valgrind test failure in replication test 1 file changed, 1 insertion(+), 1 deletion(-) -Yossi Gottlieb in commit e08c9c15: - Preserve client->id for blocked clients. - 1 file changed, 4 insertions(+), 1 deletion(-) +Oran Agra in commit 5e17e6276: + add regression test for the race in #7205 + 1 file changed, 52 insertions(+) -zhaozhao.zz in commit c6b1252f: - aof: enhance AOF_FSYNC_EVERYSEC, more details in #5985 - 2 files changed, 32 insertions(+), 3 deletions(-) +antirez in commit 96e7c011e: + Improve the PSYNC2 test reliability. + 1 file changed, 33 insertions(+), 15 deletions(-) -David Carlier in commit ce54e299: - build fix - 1 file changed, 1 insertion(+) +================================================================================ +Redis 6.0.3 Released Sat May 16 18:10:21 CEST 2020 +================================================================================ -yongman in commit c9274498: - Fix memleak in bitfieldCommand - 1 file changed, 8 insertions(+), 2 deletions(-) +Upgrade urgency CRITICAL: a crash introduced in 6.0.2 is now fixed. -James Rouzier in commit 635d8d83: - Fix start and end key initialize - 1 file changed, 2 insertions(+), 2 deletions(-) +1eab62f7e Remove the client from CLOSE_ASAP list before caching the master. -Salvatore Sanfilippo in commit 7c23e534: - Merge pull request #6047 from abhaynahar/removed-obsolete-warning-5.0 -abhay in commit 9ea8ec42: - removed obsolete warning as per - https://github.com/antirez/redis/issues/5291 - 1 file changed, 1 insertion(+), 7 deletions(-) +================================================================================ +Redis 6.0.2 Released Fri May 15 22:24:36 CEST 2020 +================================================================================ -antirez in commit 1b7407fa: - Aesthetic change to #5962 to conform to Redis style. - 1 file changed, 1 insertion(+), 3 deletions(-) +Upgrade urgency MODERATE: many not critical bugfixes in different areas. + Critical fix to client side caching when + keys are evicted from the tracking table but + no notifications are sent. -Oran Agra in commit 3bbf9747: - slave corrupts replication stream when module blocked client uses large reply (or POSTPONED_ARRAY) - 3 files changed, 15 insertions(+), 6 deletions(-) +The following are the most serious fix: -================================================================================ -Redis 5.0.4 Released Mon Mar 18 17:12:53 CET 2019 -================================================================================ +* XPENDING should not update consumer's seen-time +* optimize memory usage of deferred replies - fixed +* Fix CRC64 initialization outside the Redis server itself. +* stringmatchlen() should not expect null terminated strings. +* Cluster nodes availability checks improved when there is + high Pub/Sub load on the cluster bus. +* Redis Benchmark: Fix coredump because of double free +* Tracking: send eviction messages when evicting entries. +* rax.c updated from upstream antirez/rax. +* fix redis 6.0 not freeing closed connections during loading. -Upgrade urgency HIGH: This release fixes several Redis stability issues. +New features: -Dear Redis users, this release includes a number of fixes for bugs that may -result in Redis crashing in special conditions (not normal usage, but specific -artificial conditions), fixes to certain Redis behaviors especially around -Redis streams, and finally a set of new APIs for Redis Modules. +* Support setcpuaffinity on linux/bsd +* Client Side Caching: Add Tracking Prefix Number Stats in Server Info +* Add --user argument to redis-benchmark.c (ACL) -Specifically: +Full list of commits: -* Hyperloglog different coding errors leading to potential crashes were fixed. -* A replication bug leading to a potential crash in case of plain misuse of handshake commands was fixed. -* XCLAIM command incrementing of number of deliveries was fixed. -* LFU field management in objects was improved. -* A potential overflow in the redis-check-aof was fixed. -* A memory leak in case of API misuse was fixed. -* ZPOP* behavior when count is 0 is fixed. -* A few redis-cli --cluster bugs were fixed, plus a few improvements. -* Many other smaller bugs. +Yossi Gottlieb in commit 16ba33c05: + TLS: Fix test failures on recent Debian/Ubuntu. + 1 file changed, 20 deletions(-) -We suggest to upgrade Redis, especially in case your instance is facing -untrusted users (for instance Cloud providers) because several of these -bugs could result in unwanted crashes. +Yossi Gottlieb in commit 77ae66930: + TLS: Add crypto locks for older OpenSSL support. + 1 file changed, 45 insertions(+) -This is the list of commits: +David Carlier in commit 389697988: + NetBSD build update. + 3 files changed, 30 insertions(+), 1 deletion(-) -antirez in commit 84bdd440: - HyperLogLog: fix comment in hllCount(). - 1 file changed, 2 insertions(+), 2 deletions(-) +Madelyn Olson in commit 2435341d7: + Added a refcount on timer events to prevent deletion of recursive timer calls + 2 files changed, 12 insertions(+) -antirez in commit ef1833b3: - HyperLogLog: handle wrong offset in the base case. - 1 file changed, 2 insertions(+), 6 deletions(-) +antirez in commit 80c906bd3: + Cache master without checking of deferred close flags. + 3 files changed, 11 insertions(+), 8 deletions(-) -antirez in commit 623afd5e: - HyperLogLog: speedup fuzz test. - 1 file changed, 1 insertion(+), 2 deletions(-) +antirez in commit 74249be4a: + Track events processed while blocked globally. + 5 files changed, 32 insertions(+), 17 deletions(-) -antirez in commit 12b5ff10: - HyperLogLog: enlarge reghisto variable for safety. - 1 file changed, 6 insertions(+), 1 deletion(-) +antirez in commit 8bf660af9: + Some rework of #7234. + 4 files changed, 77 insertions(+), 65 deletions(-) -antirez in commit 254d897e: - HyperLogLog: dense/sparse repr parsing fuzz test. - 1 file changed, 29 insertions(+) +Oran Agra in commit 9da134cd8: + fix redis 6.0 not freeing closed connections during loading. + 3 files changed, 133 insertions(+), 58 deletions(-) -John Sully in commit 7f79849c: - Fix hyperloglog corruption - 1 file changed, 6 insertions(+) +antirez in commit f7f219a13: + Regression test for #7249. + 1 file changed, 22 insertions(+) -Brad Solomon in commit 3ef2c831: - Provide an uninstall target in Makefile - 1 file changed, 3 insertions(+) +antirez in commit 693629585: + rax.c updated from upstream antirez/rax. + 1 file changed, 4 insertions(+), 2 deletions(-) -antirez in commit 57aea463: - redis-check-aof: fix potential overflow. - 1 file changed, 2 insertions(+), 2 deletions(-) +antirez in commit e3b5648df: + Tracking: send eviction messages when evicting entries. + 2 files changed, 29 insertions(+), 12 deletions(-) -antirez in commit ba5145b8: - Fix objectSetLRUOrLFU() when LFU underflows. - 1 file changed, 11 insertions(+), 7 deletions(-) +Oran Agra in commit 5c41802d5: + fix unstable replication test + 1 file changed, 2 insertions(+), 2 deletions(-) -antirez in commit 76c59f0e: - Fix ZPOP return type when COUNT=0. Related to #5799. +ShooterIT in commit a23cdbb94: + Redis Benchmark: Fix coredump because of double free 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 1c636714: - Improve comments after merging #5834. - 2 files changed, 14 insertions(+), 8 deletions(-) - -Guy Benoish in commit 6a3fca4c: - Trim SDS free space of retained module strings - 4 files changed, 28 insertions(+), 5 deletions(-) +antirez in commit 1276058ea: + Cluster: clarify we always resolve the sender. + 1 file changed, 3 insertions(+), 1 deletion(-) -Guy Benoish in commit 9ec144ea: - Fix mismatching keyspace notification classes - 2 files changed, 2 insertions(+), 2 deletions(-) +antirez in commit 002fcde3d: + Cluster: refactor ping/data delay handling. + 1 file changed, 13 insertions(+), 11 deletions(-) -Guy Benoish in commit d04b5211: - Fix zlexrangespec mem-leak in genericZrangebylexCommand - 1 file changed, 4 insertions(+), 1 deletion(-) +antirez in commit 960186a71: + Cluster: introduce data_received field. + 2 files changed, 27 insertions(+), 10 deletions(-) -Guy Benoish in commit 516f1c77: - Use memtoll() in 'CONFIG SET client-output-buffer-limit' +antirez in commit 3672875b4: + stringmatchlen() should not expect null terminated strings. 1 file changed, 2 insertions(+), 2 deletions(-) -Guy Benoish in commit 8db67a55: - Increase string2ld's buffer size (and fix HINCRBYFLOAT) - 2 files changed, 5 insertions(+), 1 deletion(-) +Brad Dunbar in commit 24e12641d: + Remove unreachable branch. + 1 file changed, 2 deletions(-) -Guy Benoish in commit db3d626b: - Check server.verbosity in RM_LogRaw - 1 file changed, 2 insertions(+) +hwware in commit c7edffbd5: + add jemalloc-bg-thread config in redis conf + 1 file changed, 3 insertions(+) -Guy Benoish in commit 71439a07: - ZPOP should return an empty array if COUNT=0 - 1 file changed, 4 insertions(+), 1 deletion(-) +hwware in commit 8a9c84f4a: + add include guard for lolwut.h + 1 file changed, 6 insertions(+) -antirez in commit c8a26834: - Modules shared API: export new core APIs. - 2 files changed, 6 insertions(+) +antirez in commit cb683a84f: + Don't propagate spurious MULTI on DEBUG LOADAOF. + 2 files changed, 6 insertions(+), 3 deletions(-) -antirez in commit a13ba750: - Modules shared API: also unregister the module as user. - 1 file changed, 23 insertions(+) +antirez in commit 84d9766d6: + Dump recent backlog on master query generating errors. + 1 file changed, 29 insertions(+) + +Titouan Christophe in commit ec1e106ec: + make struct user anonymous (only typedefed) + 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 500e5117: - Modules shared API: prevent unloading of used modules. - 1 file changed, 10 insertions(+), 4 deletions(-) +antirez in commit e48c37316: + Test: --dont-clean should do first cleanup. + 1 file changed, 2 insertions(+), 5 deletions(-) -antirez in commit 7854daa1: - Modules shared API: unregister APIs function. - 1 file changed, 25 insertions(+) +Benjamin Sergeant in commit 1e561cfaa: + Add --user argument to redis-benchmark.c (ACL) + 1 file changed, 15 insertions(+), 2 deletions(-) -antirez in commit d38d82af: - Modules shared API: initial core functions. - 2 files changed, 89 insertions(+), 1 deletion(-) +antirez in commit d1af82a88: + Drop not needed part from #7194. + 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 4d747bb8: - Revert shared APIs to modify the design. - 3 files changed, 120 deletions(-) +Muhammad Zahalqa in commit 897a360d0: + Fix compiler warnings on function rev(unsigned long) + 1 file changed, 3 insertions(+), 3 deletions(-) -MeirShpilraien in commit 8824b509: - added module ability to register api to be used by other modules - 3 files changed, 120 insertions(+) +antirez in commit ac316d8cc: + Move CRC64 initialization in main(). + 2 files changed, 1 insertion(+), 4 deletions(-) -zhaozhao.zz in commit 000b055b: - Streams: checkType before XGROUP CREATE - 1 file changed, 7 insertions(+), 5 deletions(-) +antirez in commit fc7bc3204: + Fix CRC64 initialization outside the Redis server itself. + 1 file changed, 3 insertions(+) -antirez in commit 9b2a0d54: - Fix BZPOP arity, backport from fix in cd2743c. - 1 file changed, 2 insertions(+), 2 deletions(-) +hwware in commit a6e55c096: + Client Side Caching: Add Tracking Prefix Number Stats in Server Info + 3 files changed, 8 insertions(+) -chendianqiang in commit 134b2582: - optimize cluster failover - 1 file changed, 1 insertion(+) +antirez in commit b062fd523: + Fix NetBSD build by fixing redis_set_thread_title() support. + 1 file changed, 4 insertions(+), 1 deletion(-) -Steve Webster in commit 1293e2a5: - Only increment delivery count if JUSTID option is omitted - 2 files changed, 18 insertions(+), 3 deletions(-) +antirez in commit 4efb25d9c: + Rework a bit the documentation for CPU pinning. + 2 files changed, 18 insertions(+), 8 deletions(-) -Steve Webster in commit 3cc4f469: - Increment delivery counter on XCLAIM unless RETRYCOUNT specified - 2 files changed, 35 insertions(+), 2 deletions(-) +zhenwei pi in commit d6436eb7c: + Support setcpuaffinity on linux/bsd + 12 files changed, 180 insertions(+), 1 deletion(-) -antirez in commit f4edd2b9: - Merge branch '5.0' of github.com:/antirez/redis into 5.0 -swilly22 in commit cedcc54e: - document additional flag of RM_GetContextFlags - 1 file changed, 3 insertions(+) +Guy Benoish in commit 3a441c7d9: + XPENDING should not update consumer's seen-time + 4 files changed, 33 insertions(+), 20 deletions(-) -swilly22 in commit 26e98da2: - Extend REDISMODULE_CTX_FLAGS to indicate if command was sent by master - 2 files changed, 6 insertions(+) +Oran Agra in commit 75addb4fe: + optimize memory usage of deferred replies - fixed + 1 file changed, 29 insertions(+) -Salvatore Sanfilippo in commit 0e910939: - Merge pull request #5879 from meierfra-ergon/redis-cli-assume-yes -antirez in commit 67452e91: - Make comment in #5911 stay inside 80 cols. - 1 file changed, 2 insertions(+), 1 deletion(-) +Deliang Yang in commit c57d9146f: + reformat code + 1 file changed, 1 insertion(+), 1 deletion(-) -John Sully in commit 30f666ef: - Replicas aren't allowed to run the replicaof command - 1 file changed, 8 insertions(+) +Oran Agra in commit 3d3861dd8: + add daily github actions with libc malloc and valgrind + 5 files changed, 106 insertions(+), 18 deletions(-) -Frank Meier in commit bc6c1c40: - extend use of cluster-yes option to other confimation questions - 1 file changed, 9 insertions(+) -antirez in commit 76419d8d: - Merge branch '5.0' of github.com:/antirez/redis into 5.0 -Oran Agra in commit 72ba6069: - redis-cli add support for --memkeys, fix --bigkeys for module types - 1 file changed, 132 insertions(+), 81 deletions(-) +================================================================================ +Redis 6.0.1 Released Sat May 02 00:06:07 CEST 2020 +================================================================================ -chendianqiang in commit 2ca21753: - fix replicationid will not change for server.masterhost==NULL in cluster mode when restart slave - 1 file changed, 1 insertion(+), 1 deletion(-) +Upgrade urgency HIGH: This release fixes a crash when builiding against + Libc malloc. -Salvatore Sanfilippo in commit bd7ddd79: - Merge pull request #5870 from fengweiyuan/5.0 -varianfeng in commit d13bc143: - fix corrupt_rdb.c bug.Let the name of input rdb file name be valid. - 1 file changed, 2 insertions(+), 1 deletion(-) +Here we revert 8110ba888, an optimization that causes a crash due to a +bug in the code. It does not happen with the default allocator because of +differences between Jemalloc and libc malloc, so this escaped all our +testing but was reported by a user. We'll add back the original optimization +that was reverted here later, after checking what happens: it is not a +critical optimization. -artix in commit 44c5bce0: - Cluster Manager: fix replica assigment anti-affinity (create) - 1 file changed, 6 insertions(+) +The other commits are minor stuff: -artix in commit f066e526: - Cluster Manager: remove unused code elements - 1 file changed, 8 insertions(+), 13 deletions(-) +antirez in commit db73d0998: + Cast printf() argument to the format specifier. + 1 file changed, 3 insertions(+), 1 deletion(-) -Zhicheng Wei in commit 23214966: - fix clusterManagerGetAntiAffinityScore double free otypes - 1 file changed, 2 insertions(+), 1 deletion(-) +antirez in commit 7c0fe7271: + Revert "optimize memory usage of deferred replies" + 1 file changed, 31 deletions(-) -antirez in commit 80bccd71: - Remove debugging printf from replication.tcl test. - 1 file changed, 1 deletion(-) +antirez in commit 8fe25edc7: + Save a call to stopThreadedIOIfNeeded() for the base case. + 1 file changed, 3 insertions(+), 3 deletions(-) ================================================================================ -Redis 5.0.3 Released Tue Dec 11 18:17:26 CET 2018 +Redis 6.0.0 GA Released Thu Apr 30 14:55:02 CEST 2020 ================================================================================ -Upgrade urgency HIGH: Redis 5 is consolidating, upgrading is a good idea. - However there is nothing very critical here, but certain - issues resolved could lead to very rare crashes. +Upgrade urgency CRITICAL: many bugs fixed compared to the last release + candidate. Better to upgrade if you see things + affecting your environment in the changelog. + +Hi all, finally we have Redis 6.0.0 GA! Enjoy this new Redis release. +Most of the documentation was updated today so that you can likely +find what you are looking for about the new features at redis.io. +This is the list of what changed compared to the previoius release candidate: + +* XCLAIM AOF/replicas propagation fixed. +* Client side caching: new NOLOOP option to avoid getting notified about + changes performed by ourselves. +* ACL GENPASS now uses HMAC-SHA256 and have an optional "bits" argument. + It means you can use it as a general purpose "secure random strings" + primitive! +* Cluster "SLOTS" subcommand memory optimization. +* The LCS command is now a subcommand of STRALGO. +* Meaningful offset for replicas as well. More successful partial + resynchronizations. +* Optimize memory usage of deferred replies. +* Faster CRC64 algorithm for faster RDB loading. +* XINFO STREAM FULL, a new subcommand to get the whole stream state. +* CLIENT KILL USER . +* MIGRATE AUTH2 option, for ACL style authentication support. +* Other random bugfixes. + +Enjoy Redis 6! :-) +Goodbye antirez + +List of commits in this release: + +antirez in commit 1f9b82bd5: + Update help.h again before Redis 6 GA. + 1 file changed, 17 insertions(+), 12 deletions(-) + +antirez in commit 3fcffe7d0: + redis-cli: fix hints with subcommands. + 1 file changed, 2 insertions(+), 1 deletion(-) + +antirez in commit 455d8a05c: + redis-cli command help updated. + 1 file changed, 165 insertions(+), 25 deletions(-) + +zhaozhao.zz in commit 70287bbc9: + lazyfree & eviction: record latency generated by lazyfree eviction + 1 file changed, 18 insertions(+), 13 deletions(-) -Welcome to Redis 5.0.3, several interesting bug fixes here: +antirez in commit 7be21139a: + MIGRATE AUTH2 for ACL support. + 1 file changed, 19 insertions(+), 5 deletions(-) -* Redis no longer panics when you send data to a replica-mode connection that - is in MONITOR or SYNC mode. +antirez in commit e1ee1a49d: + CLIENT KILL USER . + 1 file changed, 11 insertions(+) -* Fixes to certain sorted set edge cases. You are unlikely to ever notice those - issues, but now it is more correct. +antirez in commit d56f058c0: + Fix tracking table max keys option in redis.conf. + 1 file changed, 12 insertions(+), 9 deletions(-) -* Certain BSD variants now are better supported: build & register logging - on crash. +antirez in commit 96dd5fc93: + redis-cli: safer cluster fix with unreachalbe masters. + 1 file changed, 26 insertions(+), 1 deletion(-) -* The networking core now recovers if an IPv6 address is listed in bind but - is actually not able to work because there is no such protocol in the - system. +antirez in commit 5b59d9c5d: + redis-cli: simplify cluster nodes coverage display. + 1 file changed, 10 insertions(+), 17 deletions(-) -* redis-cli cluster mode improved in many ways. Especially the fix subcommand - work was enhanced to cover other edge cases that were still not covered - after the work done for Redis 5. +antirez in commit c163d4add: + redis-cli: try to make clusterManagerFixOpenSlot() more readable. + 1 file changed, 25 insertions(+), 6 deletions(-) -* MEMORY USAGE is now more accurate. +Guy Benoish in commit aab74b715: + XINFO STREAM FULL should have a default COUNT of 10 + 1 file changed, 8 insertions(+), 4 deletions(-) -* DEBUG DIGEST-VALUE added in case you want to make sure a given set of keys - (and not the whole DB) are excatly the same between two instances. +antirez in commit 606134f9d: + Comment clearly why we moved some code in #6623. + 1 file changed, 4 insertions(+), 1 deletion(-) -* Fix a potential crash in the networking code related to recent changes - to the way the reply is consumed. +srzhao in commit ee627bb66: + fix pipelined WAIT performance issue. + 1 file changed, 13 insertions(+), 13 deletions(-) -* Reject EXEC containing write commands against an instance that changed role - from master to replica during our transaction. +antirez in commit 47b8a7f9b: + Fix create-cluster BIN_PATH. + 1 file changed, 1 insertion(+), 1 deletion(-) -* Fix a crash in KEYS and other commands using pattern matching, in an edge - case where the pattern contains a zero byte. +Guy Benoish in commit 6c0bc608a: + Extend XINFO STREAM output + 2 files changed, 226 insertions(+), 34 deletions(-) -* Fix eviction during AOF loading due to maxmemory triggered by commands - executed in loading state. +hwware in commit 5bfc18950: + Fix not used marco in cluster.c + 1 file changed, 1 insertion(+), 1 deletion(-) -The following is the list of commmits if you want to check credits or dig -further in the details. +Itamar Haber in commit 56d628f85: + Update create-cluster + 1 file changed, 1 insertion(+), 1 deletion(-) -commit 2c6ee0f9b3d9ca48c6da8bd18796186784216bff -Author: antirez -Date: Wed Dec 12 11:37:15 2018 +0100 +Itamar Haber in commit cac9d7cf7: + Adds `BIN_PATH` to create-cluster + 1 file changed, 8 insertions(+), 6 deletions(-) - freeMemoryIfNeeded() small refactoring. +Oran Agra in commit b712fba17: + hickup, re-fix dictEncObjKeyCompare + 1 file changed, 4 insertions(+), 4 deletions(-) - Related to issue #5686 and PR #5689. +Oran Agra in commit ea63aea72: + fix loading race in psync2 tests + 3 files changed, 15 insertions(+), 1 deletion(-) -commit 107e93e75acfd5def0252efb6870751940816395 -Author: zhaozhao.zz -Date: Wed Dec 12 00:25:24 2018 +0800 +antirez in commit 64e588bfa: + Rework comment in dictEncObjKeyCompare(). + 1 file changed, 8 insertions(+), 9 deletions(-) - evict: don't care about mem if loading +Oran Agra in commit 0d1e8c93b: + allow dictFind using static robj + 1 file changed, 9 insertions(+), 4 deletions(-) - When loading data, we call processEventsWhileBlocked - to process events and execute commands. - But if we are loading AOF it's dangerous, because - processCommand would call freeMemoryIfNeeded to evict, - and that will break data consistency, see issue #5686. +Madelyn Olson in commit a1bed447b: + Added crcspeed library + 2 files changed, 341 insertions(+) -antirez in commit ee93dc0b: - Crashing is too much in addReplyErrorLength(). - 1 file changed, 6 deletions(-) +Madelyn Olson in commit a75fa3aad: + Made crc64 test consistent + 1 file changed, 3 insertions(+), 2 deletions(-) -hdmg in commit c55254a5: - fix comments fault discription - 1 file changed, 1 insertion(+), 1 deletion(-) +Madelyn Olson in commit 52c75e9db: + Implemented CRC64 based on slice by 4 + 5 files changed, 124 insertions(+), 157 deletions(-) -lsytj0413 in commit dfd25013: - fix a typo: craeted -> created - 1 file changed, 1 insertion(+), 1 deletion(-) +Oran Agra in commit 8110ba888: + optimize memory usage of deferred replies + 1 file changed, 31 insertions(+) -antirez in commit 392a2566: - stringmatchlen() fuzz test added. - 3 files changed, 22 insertions(+) +Oran Agra in commit e4d2bb62b: + Keep track of meaningful replication offset in replicas too + 5 files changed, 212 insertions(+), 92 deletions(-) -antirez in commit 7602f695: - Fix stringmatchlen() read past buffer bug. +antirez in commit fea9788cc: + Fix STRALGO command flags. 1 file changed, 1 insertion(+), 1 deletion(-) -zhaozhao.zz in commit c4f3585e: - multi: ignore multiState's cmd_flags when loading AOF +Dave-in-lafayette in commit 2144047e1: + fix for unintended crash during panic response 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit d037e987: - Reject EXEC containing write commands against RO replica. - 2 files changed, 20 insertions(+) - -artix in commit e00ab324: - Cluster Manager: - Multiple owners checking in 'fix'/'check' commands is - now optional (using --cluster-search-multiple-owners). - Updated help. - 1 file changed, 14 insertions(+), 5 deletions(-) - -artix in commit 94f64de3: - Cluster Manager: FixOpenSlot now correctly updates in-memory cluster - configuration. Improved output messages. - 1 file changed, 17 insertions(+), 5 deletions(-) - -artix in commit 752d636f: - Cluster Manager: 'fix' command now handles open slots with migrating state - in one node and importing state in multiple nodes. - 1 file changed, 74 insertions(+), 6 deletions(-) - -artix in commit 552091f9: - Cluster Manager: setting new slot owner is now handled atomically in - 'fix' command. - 1 file changed, 72 insertions(+), 31 deletions(-) - -artix in commit 2280f4f7: - Cluster Manager: code cleanup. - 1 file changed, 41 insertions(+), 87 deletions(-) - -artix in commit e084b8cc: - Cluster Manager: check/fix commands now handle multiple owners even - if all slots are covered and not open. - 1 file changed, 129 insertions(+), 6 deletions(-) - -zhaozhao.zz in commit fa726e2a: - remove useless tryObjectEncoding in debug assert - 1 file changed, 1 deletion(-) - -Oran Agra in commit 40244b10: - fix #5580, display fragmentation and rss overhead bytes as signed - 2 files changed, 6 insertions(+), 6 deletions(-) - -zhaozhao.zz in commit beab3151: - networking: current_client should not be NULL when trim qb_pos +Guy Benoish in commit 43329c9b6: + Add the stream tag to XSETID tests 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 07ccb642: - Remove no longer relevant comment in processCommand(). - 1 file changed, 2 insertions(+), 6 deletions(-) +Dave-in-lafayette in commit 1e17d3de7: + fix for crash during panic before all threads are up + 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 60fdaf07: - DEBUG DIGEST-VALUE implemented. - 1 file changed, 17 insertions(+), 3 deletions(-) +antirez in commit 3722f89f4: + LCS -> STRALGO LCS. + 4 files changed, 28 insertions(+), 15 deletions(-) -antirez in commit 48b31b0d: - DEBUG DIGEST refactoring: extract function to digest a value. - 1 file changed, 142 insertions(+), 131 deletions(-) +antirez in commit 373ae6061: + Also use propagate() in streamPropagateGroupID(). + 1 file changed, 11 insertions(+), 1 deletion(-) -yura in commit ef3ff402: - redis-cli reshard/rebalance: ability to force replacement on existing keys - 1 file changed, 6 insertions(+), 5 deletions(-) +yanhui13 in commit f03f1fad6: + add tcl test for cluster slots + 1 file changed, 44 insertions(+) -Thomas Orozco in commit ee223fb8: - cli: pass auth through REDISCLI_AUTH - 1 file changed, 14 insertions(+) +yanhui13 in commit 374ffdf1c: + optimize the output of cluster slots + 1 file changed, 7 insertions(+), 4 deletions(-) -yongman in commit 41295e55: - Fix cluster call reply format readable - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit 4db38d2ef: + Minor aesthetic changes to #7135. + 1 file changed, 5 insertions(+), 7 deletions(-) -Oran Agra in commit 0ed3970f: - fix small test suite race conditions - 3 files changed, 11 insertions(+) +Valentino Geron in commit f0a261448: + XREADGROUP with NOACK should propagate only one XGROUP SETID command + 1 file changed, 13 insertions(+), 7 deletions(-) -zhaozhao.zz in commit 605dddbb: - MEMORY command: make USAGE more accurate - 1 file changed, 7 insertions(+), 6 deletions(-) +antirez in commit fbdef6a9b: + ACL: re-enable command execution of disabled users. + 1 file changed, 4 deletions(-) -yongman in commit 1f43bf29: - Fix choose a random master node for slot assignment - 1 file changed, 29 insertions(+), 5 deletions(-) +antirez in commit 05a41da75: + getRandomBytes(): use HMAC-SHA256. + 1 file changed, 30 insertions(+), 10 deletions(-) -Weiliang Li in commit 69f0c678: - fix comment typo in util.c - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit 345c3768d: + ACL GENPASS: take number of bits as argument. + 1 file changed, 21 insertions(+), 6 deletions(-) -Chris Lamb in commit bc53a3ab: - Clarify the "Creating Server TCP listening socket" error. +antirez in commit 639c8a1d9: + ACL GENPASS: emit 256 bits instead of 128. 1 file changed, 1 insertion(+), 1 deletion(-) -Chris Lamb in commit fefe5460: - Don't treat unsupported protocols as fatal errors +antirez in commit 321acea03: + ACL: deny commands execution of disabled users. 1 file changed, 4 insertions(+) -David Carlier in commit a8862972: - OpenBSD support. - 3 files changed, 74 insertions(+), 1 deletion(-) - -David Carlier in commit 5e86daf9: - Backtrace/register dump on BSD. - 3 files changed, 97 insertions(+), 3 deletions(-) +Theo Buehler in commit b0920e6e8: + TLS: Fix build with SSL_OP_NO_CLIENT_RENEGOTIATION + 1 file changed, 1 insertion(+), 1 deletion(-) -Guy Benoish in commit 7c8cf5ac: - Don't call sdscmp() with shared.maxstring or shared.minstring - 2 files changed, 23 insertions(+), 9 deletions(-) +Yossi Gottlieb in commit 149b658b5: + TLS: Fix build on older verisons of OpenSSL. + 1 file changed, 2 insertions(+) -Qu Chen in commit 39e9eda3: - Add unit test for stream XCLAIM command. - 1 file changed, 48 insertions(+) +antirez in commit 06917e581: + Tracking: test expired keys notifications. + 1 file changed, 13 insertions(+) -antirez in commit 62485232: - Abort instead of crashing when loading bad stream master key. - 1 file changed, 3 insertions(+) +antirez in commit e434b2ce4: + Tracking: NOLOOP tests. + 1 file changed, 32 insertions(+) -Madelyn Olson in commit a5487309: - Fixed a serverPanic when sending an invalid command to a monitor client - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit f3a172887: + Tracking: signal key as modified when evicting. + 1 file changed, 1 insertion(+) -================================================================================ -Redis 5.0.2 Released Thu Nov 22 11:22:37 CET 2018 -================================================================================ +antirez in commit e63bb7ec8: + Tracking: NOLOOP further implementation and fixes. + 2 files changed, 21 insertions(+), 6 deletions(-) -Upgrade urgency: CRITICAL if you use streams and consumer groups. - HIGH if you use redis-cli with Redis Cluster. - LOW otherwise. +antirez in commit 6791ff052: + Tracking: NOLOOP internals implementation. + 17 files changed, 174 insertions(+), 112 deletions(-) -Welcome to Redis 5.0.2. This release fixes two issues with Streams consumer -groups, where items could be returned duplicated by XREADGROUP when accessing -the history, and another bug where XREADGROUP can report some history even -if the comsumer pending list is empty. Both problems were addressed and unit -tests to avoid regressions implemented. Moreover this release fixes some -issue with redis-cli when in cluster mode. Finally some FreeBSD and DragonFly -build problems are now resolved. The list of the commits is below. +antirez in commit 725b8cc68: + Implement redis_set_thread_title for MacOS. + 1 file changed, 6 insertions(+) -Enjoy, -Salvatore +zhenwei pi in commit 3575b8706: + Threaded IO: set thread name for redis-server + 3 files changed, 28 insertions(+) -David Carlier in commit e8b4291a: - DragonFlyBSD little build fix - 2 files changed, 6 insertions(+), 1 deletion(-) +antirez in commit a76c67578: + Sentinel: small refactoring of sentinelCollectTerminatedScripts(). + 1 file changed, 1 insertion(+), 2 deletions(-) -yongman in commit 8fcfd374: - skip slave nodes when sending cluster setslot command +omg-by in commit 3a27064c4: + fix(sentinel): sentinel.running_scripts will always increase more times and not reset 1 file changed, 1 insertion(+) -yongman in commit d7089ddd: - Fix pointer access and memory leak in redis-cli. - 1 file changed, 6 insertions(+), 3 deletions(-) +antirez in commit 5c4c73e2c: + A few comments and name changes for #7103. + 1 file changed, 13 insertions(+), 4 deletions(-) -antirez in commit 17b4cd83: - Test: regression test for #5570. - 1 file changed, 15 insertions(+) +Oran Agra in commit 6148f9493: + testsuite run the defrag latency test solo + 3 files changed, 42 insertions(+), 2 deletions(-) -antirez in commit 45123169: - Stream: fix XREADGROUP history reading of deleted messages. - 1 file changed, 1 insertion(+), 1 deletion(-) +Jamie Scott in commit 51d3012d4: + Adding acllog-max-len to Redis.conf + 1 file changed, 9 insertions(+) -David Carlier in commit 5ad588f0: - only FreeBSD change/little warning addressing - 2 files changed, 7 insertions(+), 4 deletions(-) +antirez in commit c39f16c42: + Fix XCLAIM propagation in AOF/replicas for blocking XREADGROUP. + 2 files changed, 8 insertions(+), 3 deletions(-) -David Carlier in commit 11801e1a: - tweak form feedback - 1 file changed, 1 insertion(+), 1 deletion(-) +================================================================================ +Redis 6.0-rc4 Released Thu Apr 16 16:10:35 CEST 2020 +================================================================================ -David Carlier in commit c1f13575: - allow flavors - 1 file changed, 1 insertion(+), 1 deletion(-) +Upgrade urgency LOW: If you are using RC3 without issues, don't rush. -David Carlier in commit 275a2d49: - Fix clang build. - 1 file changed, 5 insertions(+), 1 deletion(-) +Hi all, this the latest release candidate of Redis 6. This is likely to +be very similar to what you'll see in Redis 6 GA. Please test it and +report any issue :-) -antirez in commit 44ad5141: - Test: regression test for #5577. - 1 file changed, 24 insertions(+) +Main changes in this release: -antirez in commit c7951f43: - Streams: fix XREADGROUP history reading when CG last_id is low. - 1 file changed, 12 insertions(+), 9 deletions(-) + * Big INFO speedup when using a lot of of clients. + * Big speedup on all the blocking commands: now blocking + on the same key is O(1) instead of being O(N). + * Stale replicas now allow MULTI/EXEC. + * New command: LCS (Longest Common Subsequence). + * Add a new configuration to make DEL like UNLINK. + * RDB loading speedup. + * Many bugs fixed (see the commit messages at the end of this node) -antirez in commit a69bc5be: - t_stream.c comment resized to 80 cols. - 1 file changed, 2 insertions(+), 1 deletion(-) +See you in 14 days for Redis 6 GA. -antirez in commit 5314099d: - Redis 5 changelog: don't expect Lua replies to be ordered. - 1 file changed, 14 insertions(+), 5 deletions(-) +List of commits: -================================================================================ -Redis 5.0.1 Released Wed Nov 07 13:09:30 CET 2018 -================================================================================ +antirez in commit 9f594e243: + Update SDS to latest version. + 1 file changed, 1 insertion(+), 1 deletion(-) -Upgrade urgency: URGENT if you use Redis Streams. MODERATE otherwise. - -Hi all, this is the first patch level release of Redis 5. It contains -both fixes and improvements. Here there is a list of the major ones, however -read the commit messages at the end of the changelog if you want to know -more about the smaller things. Let's start with the new features: - -* Sentinel now supports authentication! Check the Sentinel official doc - for more info. - -* Redis-cli cluster "fix" is now able to fix a big number of clusters put - in a bad condition. Previously many corner cases were not covered. - -Now the critical fixes: - -1. Fix RESTORE mismatch reply when certain keys already expired. -2. Fix an XCLAIM non trivial issue: sometimes the command returned a wrong - entry or desynchronized the protocol. - -And now the other fixes: - -3. Stack trace generation on the Raspberry PI (and 32bit ARM) fixed. -4. Don't evict expired keys when the KEYS command is called, in order to - avoid a mass deletion event. However expired keys are not displayed - by KEYS as usually. -5. Improvements in the computation of the memory used, when estimating - the AOF buffers. -6. XRANGE COUNT of 0 fixed. -7. "key misses" stats accounting fixed. Many cache misses were not counted. -8. When in MULTI state, return OOM while accumulating commands and there - is no longer memory available. -9. Fix build on FreeBSD and possibly others. -10. Fix a crash in Redis modules, thread safe context reply accumulation. -11. Fix a race condition when producing the RDB file for full SYNC. -12. Disable protected mode in Sentinel. -13. More commands now have the HELP subcommand. -14. Fixed an issue about adaptive server HZ timer. -15. Fix cluster-replica-no-failover option name. - -Finally, this is the list of commits. Enjoy Redis 5.0.1! - -antirez in commit c801283f: - Fix cluster-replica-no-failover option name. +antirez in commit 48781dd95: + RESP3: fix HELLO map len in Sentinel mode. 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 4c4f50e1: - MEMORY command: make strcasecmp() conditional like the following. - 1 file changed, 1 insertion(+), 2 deletions(-) +antirez in commit 371ab0cff: + Don't allow empty spaces in ACL usernames. + 1 file changed, 36 insertions(+), 8 deletions(-) -Itamar Haber in commit a7b46e0e: - Uppercases subcommands in MEMORY HELP - 1 file changed, 5 insertions(+), 5 deletions(-) +antirez in commit b86140ac5: + Don't allow empty spaces in ACL key patterns. + 1 file changed, 12 insertions(+), 1 deletion(-) -Itamar Haber in commit 80e129d9: - Standardizes `MEMORY HELP` subcommand - 1 file changed, 13 insertions(+), 14 deletions(-) +liumiuyong in commit a7ee3c3e7: + FIX: truncate max/min longitude,latitude related geo_point (ex: {180, 85.05112878} ) + 1 file changed, 4 insertions(+) -valentino in commit 88805cbb: - fix short period of server.hz being uninitialized +Guy Benoish in commit e5b9eb817: + Typo in getTimeoutFromObjectOrReply's error reply 1 file changed, 1 insertion(+), 1 deletion(-) -Itamar Haber in commit 6b402733: - Adds HELP to LATENCY - 1 file changed, 14 insertions(+), 2 deletions(-) - -yongman in commit 1c637de9: - fix malloc in clusterManagerComputeReshardTable +antirez in commit 0f31bb5c1: + Fix HELLO reply in Sentinel mode, see #6160. 1 file changed, 1 insertion(+), 1 deletion(-) -artix in commit 90b52fde: - Cluster Manager: removed unused var. - 1 file changed, 1 insertion(+), 2 deletions(-) - -artix in commit 89cbb5df: - Cluster Manager: further improvements to "fix": - clusterManagerFixOpenSlot: ensure that the slot is unassigned before ADDSLOTS - clusterManagerFixSlotsCoverage: after cold migration, the slot configuration is now updated on all the nodes. - 1 file changed, 49 insertions(+), 10 deletions(-) +hwware in commit b92d9a895: + fix spelling in acl.c + 1 file changed, 2 insertions(+), 2 deletions(-) -artix in commit 175515c9: - Cluster Manager: fixed string parsing issue in clusterManagerGetConfigSignature +antirez in commit 8f896e57a: + Fix zsetAdd() top comment spelling. 1 file changed, 3 insertions(+), 3 deletions(-) -artix in commit 3997dd6e: - Cluster Manager: better fix subcommand. - 1 file changed, 78 insertions(+), 20 deletions(-) +hayleeliu in commit 8f5157058: + fix spelling mistake in bitops.c + 1 file changed, 1 insertion(+), 1 deletion(-) -artix in commit bd80291c: - Cluster Manager: fixed typos in comments. - 1 file changed, 3 insertions(+), 3 deletions(-) +antirez in commit ddeda9ceb: + Fix function names in zslDeleteNode() top comment. + 1 file changed, 2 insertions(+), 1 deletion(-) -artix in commit 4369cbce: - Cluster Manager: fixed 'DELSLOT' subcommand typo. +antirez in commit bde1f0a8e: + RESP3: change streams items from maps to arrays. 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 1ed821e2: - Fix XCLAIM missing entry bug. - 1 file changed, 3 insertions(+), 2 deletions(-) - -michael-grunder in commit b49bcd01: - Use typedef'd mstime_t instead of time_t +antirez in commit bec68bff2: + Use the special static refcount for stack objects. 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 09d1849e: - Improve streamReplyWithRange() top comment. - 1 file changed, 9 insertions(+), 5 deletions(-) +antirez in commit 0f239e51b: + RDB: refactor some RDB loading code into dbAddRDBLoad(). + 3 files changed, 22 insertions(+), 4 deletions(-) -antirez in commit bdf6306f: - Add support for Sentinel authentication. - 1 file changed, 17 insertions(+), 3 deletions(-) +antirez in commit f855db61b: + incrRefCount(): abort on statically allocated object. + 2 files changed, 12 insertions(+), 2 deletions(-) -antirez in commit 50222af5: - Disable protected mode in Sentinel mode. - 1 file changed, 1 insertion(+) +antirez in commit 23094ba01: + More powerful DEBUG RELOAD. + 3 files changed, 55 insertions(+), 16 deletions(-) -antirez in commit 643ee6e3: - When replica kills a pending RDB save during SYNC, log it. - 1 file changed, 6 insertions(+) +antirez in commit 8161a7a3e: + RDB: clarify a condition in rdbLoadRio(). + 2 files changed, 9 insertions(+), 2 deletions(-) -Andrey Bugaevskiy in commit 8b609c99: - Move child termination to readSyncBulkPayload - 1 file changed, 6 insertions(+), 7 deletions(-) +antirez in commit 61b153073: + RDB: load files faster avoiding useless free+realloc. + 7 files changed, 40 insertions(+), 28 deletions(-) -Andrey Bugaevskiy in commit 27102605: - Prevent RDB autosave from overwriting full resync results - 1 file changed, 7 insertions(+) +antirez in commit 414debfd0: + Speedup: unblock clients on keys in O(1). + 4 files changed, 50 insertions(+), 23 deletions(-) -antirez in commit a677923d: - asyncCloseClientOnOutputBufferLimitReached(): don't free fake clients. - 1 file changed, 1 insertion(+) +antirez in commit cbcd07777: + Fix ACL HELP table missing comma. + 1 file changed, 12 insertions(+), 12 deletions(-) -David Carlier in commit 427e440a: - needs it for the global +mymilkbottles in commit 2437455f2: + Judge the log level in advance 1 file changed, 1 insertion(+) -David Carlier in commit 28f9ca4e: - Fix non Linux build. - 3 files changed, 20 insertions(+), 1 deletion(-) +antirez in commit 35c64b898: + Speedup INFO by counting client memory incrementally. + 4 files changed, 52 insertions(+), 26 deletions(-) -zhaozhao.zz in commit 4bf9efe2: - MULTI: OOM err if cannot free enough memory in MULTI/EXEC context - 1 file changed, 5 insertions(+), 2 deletions(-) +qetu3790 in commit c3ac71748: + fix comments about RESIZE DB opcode in rdb.c + 1 file changed, 1 insertion(+), 4 deletions(-) -antirez in commit 4fbd7a39: - Add command fingerprint comment for XSETID. - 1 file changed, 3 insertions(+), 1 deletion(-) +antirez in commit c8dbcff9d: + Clarify redis.conf comment about lazyfree-lazy-user-del. + 1 file changed, 9 insertions(+), 5 deletions(-) -Itamar Haber in commit 2480db53: - Plugs a potential underflow - 1 file changed, 1 insertion(+) +zhaozhao.zz in commit abd5156f2: + lazyfree: add a new configuration lazyfree-lazy-user-del + 4 files changed, 7 insertions(+), 2 deletions(-) -Itamar Haber in commit e5e4d2ef: - Corrects inline documentation of syntax - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit 5719b3054: + LCS: more tests. + 1 file changed, 8 insertions(+) -zhaozhao.zz in commit 713800d2: - if we read a expired key, misses++ - 1 file changed, 5 insertions(+), 1 deletion(-) +antirez in commit c89e1f293: + LCS: allow KEYS / STRINGS to be anywhere. + 1 file changed, 6 deletions(-) + +antirez in commit 0b16f8d44: + LCS tests. + 1 file changed, 22 insertions(+) -antirez in commit e79ee263: - Fix XRANGE COUNT option for value of 0. - 1 file changed, 8 insertions(+), 2 deletions(-) +antirez in commit 9254a805d: + LCS: get rid of STOREIDX option. Fix get keys helper. + 2 files changed, 20 insertions(+), 21 deletions(-) -antirez in commit 505cc70f: - Fix typo in streamReplyWithRange() top comment. +antirez in commit a4c490703: + LCS: fix stale comment. 1 file changed, 1 insertion(+), 1 deletion(-) -Damien Tournoud in commit 3c36561d: - Overhead is the allocated size of the AOF buffer, not its length - 2 files changed, 2 insertions(+), 2 deletions(-) +antirez in commit cb92c23de: + LCS: output LCS len as well in IDX mode. + 1 file changed, 6 insertions(+), 1 deletion(-) -antirez in commit 3761582f: - Simplify part of the #5470 patch. - 1 file changed, 11 insertions(+), 12 deletions(-) +antirez in commit 56a52e804: + LCS: MINMATCHLEN and WITHMATCHLEN options. + 1 file changed, 24 insertions(+), 11 deletions(-) -zhaozhao.zz in commit edc47a3a: - do not delete expired keys in KEYS command - 1 file changed, 34 insertions(+), 27 deletions(-) +antirez in commit ebb09a5c3: + LCS: 7x speedup by accessing the array with better locality. + 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 9872af6d: - Use guide comments to make changes in #5462 more obvious. - 1 file changed, 6 insertions(+) +antirez in commit a9f8a8cba: + LCS: implement KEYS option. + 1 file changed, 18 insertions(+), 2 deletions(-) -youjiali1995 in commit 3f399c3b: - migrate: fix mismatch of RESTORE reply when some keys have expired. - 1 file changed, 8 insertions(+), 6 deletions(-) +antirez in commit 4aa24e62a: + LCS: other fixes to range emission. + 1 file changed, 20 insertions(+), 16 deletions(-) -hujie in commit eaaff621: - fix typo in config.c +antirez in commit 2b67b6b87: + LCS: fix emission of last range starting at index 0. 1 file changed, 1 insertion(+), 1 deletion(-) -hujiecs in commit 43ebb7ee: - several typos fixed, optimize MSETNX to avoid unnecessary loop - 4 files changed, 4 insertions(+), 4 deletions(-) - -antirez in commit de8fdaac: - Remove useless complexity from MSET implementation. - 1 file changed, 5 insertions(+), 7 deletions(-) +antirez in commit 420aac727: + LCS: implement range indexes option. + 1 file changed, 59 insertions(+), 9 deletions(-) -antirez in commit dc8f1112: - Fix again stack generation on the Raspberry Pi. - 1 file changed, 4 insertions(+) +antirez in commit a518a9a76: + LCS: initial functionality implemented. + 4 files changed, 156 insertions(+), 1 deletion(-) -antirez in commit 83a6e81d: - Get rid of the word slave in the release note of Redis 5. - 1 file changed, 2 insertions(+), 2 deletions(-) +srzhao in commit 026cc11b0: + Check OOM at script start to get stable lua OOM state. + 3 files changed, 11 insertions(+), 4 deletions(-) -================================================================================ -Redis 5.0.0 Released Wed Oct 17 13:28:26 CEST 2018 -================================================================================ +Oran Agra in commit 02b594f6a: + diffrent fix for runtest --host --port + 2 files changed, 13 insertions(+), 13 deletions(-) -Upgrade urgency CRITICAL: Several fixes to streams AOF and replication. - -Hi all and welcome to the first stable release of Redis 5! \o/ - -To start a quick recap of what's new in Redis 5: - -1. The new Stream data type. https://redis.io/topics/streams-intro -2. New Redis modules APIs: Timers, Cluster and Dictionary APIs. -3. RDB now store LFU and LRU information. -4. The cluster manager was ported from Ruby (redis-trib.rb) to C code - inside redis-cli. Check `redis-cli --cluster help` for more info. -5. New sorted set commands: ZPOPMIN/MAX and blocking variants. -6. Active defragmentation version 2. -7. Improvemenets in HyperLogLog implementations. -8. Better memory reporting capabilities. -9. Many commands with sub-commands now have an HELP subcommand. -10. Better performances when clients connect and disconnect often. -11. Many bug fixes and other random improvements. -12. Jemalloc was upgraded to version 5.1 -13. CLIENT UNBLOCK and CLIENT ID. -14. The LOLWUT command was added. http://antirez.com/news/123 -15. We no longer use the "slave" word if not for API backward compatibility. -16. Differnet optimizations in the networking layer. -17. Lua improvements: - - Better propagation of Lua scripts to replicas / AOF. - - Lua scripts can now timeout and get in -BUSY state in the replica as well. -18. Dynamic HZ to balance idle CPU usage with responsiveness. -19. The Redis core was refactored and improved in many ways. - -However the list above really does not do justice to the changes of Redis 5 -since the core was improved in many ways during the development of the new -version. However certain changes were back ported into Redis 4 once they were -sensed as safe, because many improvements were hard to distinguish from fixes. - -The most important user facing improvement is without doubts the introduction -of the new general purpose data type after years: the streams. - -Note that we worked to improve and fix streams till a few hours ago, so while -we are not aware of critical bugs in this release, surely there is to handle it -with some care for the first weeks. Bug reporting will be highly appreciated and -we are ready to work immediately to release 5.0.1 once there is enough important -stuff to justify a new release (probably soon). - -People not using the streams can have probably a better production-ready -experience with Redis 5, also because many internals are shared with Redis 4 -so the jump is not as big as it was between 3.2 and 4 in terms of how things -internally work. - -Well, many thanks to the Redis community and the developers that made -this release possible, contributing bug reports, patches, new features, working -on the clients, sometimes debugging problems for days. Also thank to everybody -that adopted Redis for their use cases making things work for users worldwide. - -The list of commits in this release follows. - -Cheers, -Salvatore +Guy Benoish in commit f695d1830: + Try to fix time-sensitive tests in blockonkey.tcl + 1 file changed, 54 insertions(+), 1 deletion(-) -antirez in commit bcc0916d: - Fix conditional in XGROUP. - 1 file changed, 1 insertion(+), 1 deletion(-) +Guy Benoish in commit 0e42cfc36: + Use __attribute__ only if __GNUC__ is defined + 1 file changed, 12 insertions(+), 3 deletions(-) -antirez in commit 1b2f23f3: - Update help.h for redis-cli. - 1 file changed, 57 insertions(+), 7 deletions(-) +Guy Benoish in commit 91ed9b3c4: + Modules: Perform printf-like format checks in variadic API + 1 file changed, 3 insertions(+), 3 deletions(-) -antirez in commit de0ae56c: - Tests for XGROUP CREATE MKSTREAM. - 1 file changed, 11 insertions(+) +Valentino Geron in commit 3e0d20962: + XREAD and XREADGROUP should not be allowed from scripts when BLOCK option is being used + 3 files changed, 18 insertions(+), 2 deletions(-) -antirez in commit 56c3dfa1: - Fix XGROUP CREATE MKSTREAM handling of . - 1 file changed, 7 insertions(+), 2 deletions(-) +Guy Benoish in commit 240094c9b: + Stale replica should allow MULTI/EXEC + 1 file changed, 3 insertions(+), 3 deletions(-) -antirez in commit 2687f228: - Process MKSTREAM option of XGROUP CREATE at a later time. - 1 file changed, 28 insertions(+), 17 deletions(-) +Xudong Zhang in commit 209f3a1eb: + fix integer overflow + 1 file changed, 2 insertions(+), 2 deletions(-) -zhaozhao.zz in commit cfbaf8f1: - Scripting & Streams: some commands need right flags - 1 file changed, 5 insertions(+), 5 deletions(-) +Guy Benoish in commit 024c380b9: + Fix no-negative-zero test + 1 file changed, 1 insertion(+) -antirez in commit 4e4099b9: - XGROUP CREATE: MKSTREAM option for automatic stream creation. - 1 file changed, 29 insertions(+), 5 deletions(-) +Oran Agra in commit a38ff404b: + modules don't signalModifiedKey in setKey() since that's done (optionally) in RM_CloseKey + 4 files changed, 8 insertions(+), 8 deletions(-) -zhaozhao.zz in commit 6dd4d864: - Streams: Tests modified XSTREAM -> XSETID - 1 file changed, 2 insertions(+), 2 deletions(-) +Oran Agra in commit 814874d68: + change CI to build and run the module api tests + 1 file changed, 2 insertions(+) -zhaozhao.zz in commit 3aff0e8c: - Streams: rewrite empty streams with certain lastid +Oran Agra in commit 061616c1b: + fix possible warning on incomplete struct init 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 880b563e: - Tests modified to use XADD MAXLEN 0 + XSETID. - 1 file changed, 12 insertions(+), 26 deletions(-) - -antirez in commit 83c87835: - Streams: rewrite empty streams with XADD MAXLEN 0. Use XSETID. - 1 file changed, 18 insertions(+), 12 deletions(-) +Guy Benoish in commit 7764996be: + Make sure Redis does not reply with negative zero + 2 files changed, 10 insertions(+) -antirez in commit fd22e3ac: - XSETID: accept IDs based on last entry. - 1 file changed, 18 insertions(+), 5 deletions(-) +Guy Benoish in commit eba28e2ce: + DEBUG OBJECT should pass keyname to module when loading + 3 files changed, 4 insertions(+), 4 deletions(-) -antirez in commit dfab3cba: - Streams: XSTREAM SETID -> XSETID. - 3 files changed, 17 insertions(+), 67 deletions(-) +David Carlier in commit 15c9e79a7: + debug, dump registers on arm too. + 1 file changed, 55 insertions(+), 27 deletions(-) -zhaozhao.zz in commit a3fb28ed: - Streams: rewrite id in XSTREAM CREATE * - 1 file changed, 4 insertions(+) +hwware in commit cd2b5df97: + fix spelling in cluster.c + 1 file changed, 1 insertion(+), 1 deletion(-) -zhaozhao.zz in commit f4b4db13: - Streams: add tests for aof rewrite - 1 file changed, 23 insertions(+) +Valentino Geron in commit 8cdc153f5: + XACK should be executed in a "all or nothing" fashion. + 2 files changed, 23 insertions(+), 1 deletion(-) -zhaozhao.zz in commit d22f1ef0: - Stream & AOF: rewrite stream in correct way - 1 file changed, 32 insertions(+), 16 deletions(-) +hwware in commit b35407fa7: + add check for not switching between optin optout mode directly + 1 file changed, 12 insertions(+), 1 deletion(-) -zhaozhao.zz in commit 6455274d: - Streams: add tests for XSTREAM command - 1 file changed, 39 insertions(+) +hwware in commit 4395889c9: + add check for not providing both optin optout flag + 1 file changed, 8 insertions(+) -zhaozhao.zz in commit 0edbe953: - Streams: add a new command XTREAM - 3 files changed, 67 insertions(+) +Guy Benoish in commit 1907e0f18: + PERSIST should notify a keyspace event + 1 file changed, 1 insertion(+) -Hamid Alaei in commit 9714bba2: - fix timer context selected database - 1 file changed, 3 insertions(+), 1 deletion(-) +Guy Benoish in commit c35a53169: + streamReplyWithRange: Redundant XSETIDs to replica + 1 file changed, 2 insertions(+), 1 deletion(-) -antirez in commit eb53f15a: - Make comment about nack->consumer test for minidle more obvious. - 1 file changed, 4 insertions(+), 2 deletions(-) +antirez in commit 6fe66e096: + Simplify comment in moduleTryServeClientBlockedOnKey(). + 1 file changed, 3 insertions(+), 12 deletions(-) -antirez in commit a77f836e: - Streams: use propagate_last_id itself as streamPropagateGroupID trigger. - 1 file changed, 2 insertions(+), 2 deletions(-) +Guy Benoish in commit 193fc241c: + Fix memory corruption in moduleHandleBlockedClients + 3 files changed, 149 insertions(+), 46 deletions(-) -antirez in commit 0f0610eb: - Streams: better naming: lastid_updated -> propagate_last_id. - 1 file changed, 6 insertions(+), 6 deletions(-) +================================================================================ +Redis 6.0-rc3 Released Tue Mar 31 17:42:39 CEST 2020 +================================================================================ -zhaozhao.zz in commit a745e423: - Streams: panic if streamID invalid after check, should not be possible. - 1 file changed, 2 insertions(+), 1 deletion(-) +Upgrade urgency CRITICAL: A connection management bug introduced with the + SSL implementation can crash Redis easily. + +Dear users, this is a list of the major changes in this release, please check +the list of commits for detail: + +* Fix crash due to refactoring for SSL, for the connection code. +* Precise timeouts for blocking commands. Now the timeouts have HZ + resolution regardless of the number of connected clinets. New timeouts + are stored in a radix tree and sorted by expire time. +* Fix rare crash when resizing the event loop because of CONFIG maxclients. +* Fix systemd readiness after successful partial resync. +* Redis-cli ask password mode to be prompted at startup (for additional safety). +* Keyspace notifications added to MIGRATE / RESTORE. +* Threaded I/O bugs fixed. +* Implement new ACL style AUTH in Sentinel. +* Make 'requirepass' more backward compatible with Redis <= 5. +* ACL: Handle default user as disabled if it's off regardless of "nopass". +* Fix a potential inconsistency when upgrading an instance in Redis Cluster + and restarting it. The instance will act as a replica but will actually be + set as a master immediately. However the choice of what to do with already + expired keys, on loading, was made from the POV of replicas. +* Abort transactions after -READONLY error. +* Many different fixes to module APIs. +* BITFIELD_RO added to call the command on read only replicas. +* PSYNC2: meaningful offset implementation. Allow the disconnected master + that is still sending PINGs to replicas, to be able to successfully + PSYNC incrementally to new slaves, discarding the last part of the + replication backlog consisting only of PINGs. +* Fix pipelined MULTI/EXEC during Lua scripts are in BUSY state. +* Re-fix propagation API in modules, broken again after other changes. + +antirez in commit ef1b1f01: + cast raxSize() to avoid warning with format spec. + 1 file changed, 1 insertion(+), 1 deletion(-) + +antirez in commit 9f347fab: + Minor changes to #7037. + 2 files changed, 14 insertions(+), 5 deletions(-) + +Guy Benoish in commit a509400d: + Modules: Test MULTI/EXEC replication of RM_Replicate + 6 files changed, 49 insertions(+), 9 deletions(-) + +Guy Benoish in commit 805c8c94: + RENAME can unblock XREADGROUP + 3 files changed, 25 insertions(+), 1 deletion(-) + +antirez in commit 97b80b57: + Fix the propagate Tcl test after module changes. + 1 file changed, 1 insertion(+), 1 deletion(-) + +antirez in commit 4f6b6b80: + Modify the propagate unit test to show more cases. + 1 file changed, 30 insertions(+), 2 deletions(-) -zhaozhao.zz in commit 9974be13: - Streams: propagate lastid in XCLAIM when it has effect - 1 file changed, 13 insertions(+), 6 deletions(-) +antirez in commit 616b1cb7: + Fix module commands propagation double MULTI bug. + 4 files changed, 25 insertions(+), 8 deletions(-) -zhaozhao.zz in commit 69a628d0: - Streams: XCLAIM ignore minidle if NACK is created by FORCE - 1 file changed, 4 insertions(+), 2 deletions(-) +antirez in commit 08fdef4b: + Fix RM_Call() stale comment due to cut&paste. + 1 file changed, 1 insertion(+), 3 deletions(-) -zhaozhao.zz in commit a04b43c7: - Streams: bugfix XCLAIM should propagate group name not consumer name +OMG-By in commit 26b79ca1: + fix: dict.c->dictResize()->minimal type 1 file changed, 1 insertion(+), 1 deletion(-) -Sergey Chupov in commit 8977a90c: - fixed typos in readme - 1 file changed, 2 insertions(+), 2 deletions(-) +zhaozhao.zz in commit fa418637: + PSYNC2: reset backlog_idx and master_repl_offset correctly + 1 file changed, 10 insertions(+), 5 deletions(-) -antirez in commit 3a745674: - redis.conf typo fixed: ingore -> ignore. - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit bbbc80ac: + Precise timeouts: reference client pointer directly. + 1 file changed, 13 insertions(+), 16 deletions(-) -antirez in commit 22770d76: - Rax: radix tree updated to latest version from antirez/rax. - 2 files changed, 233 insertions(+), 68 deletions(-) +antirez in commit c3b268a0: + timeout.c created: move client timeouts code there. + 5 files changed, 198 insertions(+), 167 deletions(-) -antirez in commit fbac534f: - Test: avoid time related false positive in RESTORE test. - 1 file changed, 1 insertion(+), 1 deletion(-) +Oran Agra in commit 0f7dfc37: + AOFRW on an empty stream created with MKSTREAM loads badkly + 2 files changed, 15 insertions(+), 1 deletion(-) -antirez in commit 49872337: - LOLWUT: capitalize Nees. - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit 67643ead: + Precise timeouts: cleaup the table on unblock. + 3 files changed, 21 insertions(+), 2 deletions(-) -antirez in commit 80c471f5: - Test: cgroup propagation test also for NOACK variant. - 1 file changed, 39 insertions(+), 29 deletions(-) +antirez in commit ad94066e: + Precise timeouts: fix comments after functional change. + 2 files changed, 6 insertions(+), 6 deletions(-) -antirez in commit 8defa5da: - Test: consumer group last ID slave propagation test. - 1 file changed, 39 insertions(+) +antirez in commit a443ec2e: + Precise timeouts: use only radix tree for timeouts. + 3 files changed, 15 insertions(+), 38 deletions(-) -zhaozhao.zz in commit e1e3eaca: - Avoid recreate write handler for protected client. - 1 file changed, 4 insertions(+) +antirez in commit 6862fd70: + Precise timeouts: fast exit for clientsHandleShortTimeout(). + 1 file changed, 1 insertion(+) -antirez in commit b501fd5d: - Fix propagation of consumer groups last ID. - 3 files changed, 56 insertions(+), 9 deletions(-) +antirez in commit 30f1df8c: + Precise timeouts: fix bugs in initial implementation. + 2 files changed, 5 insertions(+), 1 deletion(-) +antirez in commit 7add0f24: + Precise timeouts: working initial implementation. + 3 files changed, 110 insertions(+), 28 deletions(-) -================================================================================ -Redis 5.0-rc6 Released Wed Oct 10 11:03:54 CEST 2018 -================================================================================ +antirez in commit 9d6d1779: + Precise timeouts: refactor unblocking on timeout. + 2 files changed, 33 insertions(+), 13 deletions(-) -Upgrade urgency HIGH: Many bugs fixed especially in the context of streams. +antirez in commit 316a8f15: + PSYNC2: fix backlog_idx when adjusting for meaningful offset + 1 file changed, 3 insertions(+) -This is probably the last release candidate of Redis 5. The Redis 5 GA version -will be released 17th of October. The main highlights of this release are: +伯成 in commit 11db53f8: + Boost up performance for redis PUB-SUB patterns matching + 3 files changed, 43 insertions(+), 11 deletions(-) -* Critical AOF bug, as old as AOF itself: if an open MULTI/EXEC block is at - the end of the AOF file, Redis would still read the half-transaction when - reloading back the AOF. -* The slave name was removed from logs and documentation, now replica is used - instead. -* LOLWUT command added. -* New modules APIs: Disable Redis Cluster redirection. -* New modules APIs: Sorted dictionaries data type. -* Modules APIs fixes: timer / cluster messages callback now can call RM_Call(). -* Fix for #5024 - commandstats for multi-exec were logged as EXEC. -* A number of optimizations and fixes for the stream data type. -* Many other stability improvements. +antirez in commit e257f121: + PSYNC2: meaningful offset test. + 2 files changed, 62 insertions(+) -This is the list of comments and contributors: +antirez in commit 5f72f696: + PSYNC2: meaningful offset implemented. + 3 files changed, 40 insertions(+), 1 deletion(-) -antirez in commit 9a6fa7d0: - changelog.tcl: get optional argument for number of commits. - 1 file changed, 8 insertions(+), 3 deletions(-) +antirez in commit 8caa2714: + Explain why we allow transactions in -BUSY state. + 1 file changed, 9 insertions(+), 2 deletions(-) -antirez in commit 101e419f: - Free protected clients asynchronously. - 1 file changed, 7 insertions(+) +Oran Agra in commit e43cd831: + MULTI/EXEC during LUA script timeout are messed up + 2 files changed, 73 insertions(+) -antirez in commit 726debb8: - Actually use the protectClient() API where needed. - 2 files changed, 8 insertions(+), 9 deletions(-) - -antirez in commit 0b87f78a: - Introduce protectClient() + some refactoring. - 2 files changed, 60 insertions(+), 18 deletions(-) +antirez in commit 34b89832: + Improve comments of replicationCacheMasterUsingMyself(). + 1 file changed, 6 insertions(+), 1 deletion(-) -zhaozhao.zz in commit 6aa8ac70: - debug: avoid free client unexpectedly when reload & loadaof - 1 file changed, 8 insertions(+), 2 deletions(-) +antirez in commit 70a98a43: + Fix BITFIELD_RO test. + 2 files changed, 5 insertions(+), 5 deletions(-) -antirez in commit 48040b02: - aof.c: improve indentation and change warning message. - 1 file changed, 11 insertions(+), 4 deletions(-) +antirez in commit 8783304a: + Abort transactions after -READONLY error. Fix #7014. + 1 file changed, 1 insertion(+) -zhaozhao.zz in commit 7cc20569: - AOF: discard if we lost EXEC when loading aof - 2 files changed, 14 insertions(+), 3 deletions(-) +antirez in commit ec9cf002: + Minor changes to BITFIELD_RO PR #6951. + 1 file changed, 9 insertions(+), 6 deletions(-) -antirez in commit 2007d30c: - Refactoring of XADD / XTRIM MAXLEN rewriting. - 1 file changed, 15 insertions(+), 22 deletions(-) +bodong.ybd in commit b3e4abf0: + Added BITFIELD_RO variants for read-only operations. + 4 files changed, 54 insertions(+), 1 deletion(-) -zhaozhao.zz in commit 6a298110: - Streams: add test cases for XADD/XTRIM maxlen - 1 file changed, 46 insertions(+) +antirez in commit 50f8f950: + Modules: updated function doc after #7003. + 1 file changed, 6 insertions(+), 1 deletion(-) -zhaozhao.zz in commit 041161b7: - Streams: propagate specified MAXLEN instead of approximated - 1 file changed, 35 insertions(+), 6 deletions(-) +Guy Benoish in commit f2f3dc5e: + Allow RM_GetContextFlags to work with ctx==NULL + 1 file changed, 16 insertions(+), 14 deletions(-) -zhaozhao.zz in commit f04d799b: - Streams: reset approx_maxlen in every maxlen loop +hwware in commit eb808879: + fix potentical memory leak in redis-cli 1 file changed, 2 insertions(+) -zhaozhao.zz in commit affd9365: - Streams: XTRIM will return an error if MAXLEN with a count < 0 - 1 file changed, 6 insertions(+), 1 deletion(-) +Yossi Gottlieb in commit cdcab0e8: + Fix crashes related to failed/rejected accepts. + 1 file changed, 6 insertions(+), 5 deletions(-) -zhaozhao.zz in commit 4c405ad0: - Streams: propagate original MAXLEN argument in XADD context - 1 file changed, 3 insertions(+), 12 deletions(-) +Yossi Gottlieb in commit 50dcd9f9: + Cluster: fix misleading accept errors. + 1 file changed, 4 insertions(+), 3 deletions(-) -antirez in commit 5c6d4b4a: - Fix typo in replicationCron() comment. - 1 file changed, 1 insertion(+), 1 deletion(-) +Yossi Gottlieb in commit 87dbd8f5: + Conns: Fix connClose() / connAccept() behavior. + 3 files changed, 48 insertions(+), 32 deletions(-) -antirez in commit a67a8dbf: - Fix typo in design comment of bio.c. +hwware in commit 81e8686c: + remove redundant Semicolon 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit c4ab5a05: - xclaimCommand(): fix comment typos. - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit dc0b628a: - streamAppendItem(): Update the radix tree pointer only if changed. - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 4566fbc7: - Listpack: optionally force reallocation on inserts. - 1 file changed, 20 insertions(+) - -antirez in commit 5eca170c: - Fix printf type mismatch in genRedisInfoString(). +hwware in commit c7524a7e: + clean CLIENT_TRACKING_CACHING flag when disabled caching 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 260b53a2: - streamIteratorRemoveEntry(): set back lp only if pointer changed. +hwware in commit 2dd1ca6a: + add missing commands in cluster help 1 file changed, 2 insertions(+), 1 deletion(-) -zhaozhao.zz in commit 5d12f9d9: - Streams: update listpack with new pointer in XDEL - 1 file changed, 3 insertions(+) - -zhaozhao.zz in commit 6b7ad838: - bugfix: replace lastcmd with cmd when rewrite BRPOPLPUSH as RPOPLPUSH - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit 3454a043: - script cache memory in INFO and MEMORY includes both script code and overheads - 2 files changed, 3 insertions(+), 3 deletions(-) - -Oran Agra in commit d6aeca86: - fix #5024 - commandstats for multi-exec were logged as EXEC. - 2 files changed, 63 insertions(+), 2 deletions(-) - -antirez in commit a996b2a2: - Fix XINFO comment for consistency. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Bruce Merry in commit 1a8447b6: - Fix invalid use of sdsZmallocSize on an embedded string - 1 file changed, 1 insertion(+), 1 deletion(-) - -Bruce Merry in commit 8dde46ad: - Fix incorrect memory usage accounting in zrealloc - 3 files changed, 24 insertions(+), 2 deletions(-) - -Hamid Alaei in commit b362a1b7: - fix dict get on not found - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 55e9df8a: - Try to avoid issues with GCC pragmas and older compilers. - 1 file changed, 7 insertions(+), 4 deletions(-) - -antirez in commit b0d22702: - Modules: hellodict example WIP #3: KEYRANGE. - 1 file changed, 40 insertions(+) - -antirez in commit af2f6682: - Modules: Modules: dictionary API WIP #13: Compare API exported. - 2 files changed, 6 insertions(+) - -antirez in commit f9a3e6ef: - Modules: Modules: dictionary API WIP #12: DictCompare API. - 1 file changed, 8 insertions(+) - -antirez in commit 01e0341a: - Modules: Modules: dictionary API WIP #11: DictCompareC API. - 1 file changed, 18 insertions(+) - -antirez in commit f9b3ce9a: - Modules: hellodict example WIP #1: GET command. - 1 file changed, 18 insertions(+) - -antirez in commit 36e66d86: - Modules: hellodict example WIP #1: SET command. - 1 file changed, 74 insertions(+) - -antirez in commit e33fdbe8: - Modules: remove useless defines in hellotimer.c - 2 files changed, 6 insertions(+), 4 deletions(-) - -antirez in commit 1c8b2248: - Modules: fix top comment of hellotimer.c - 1 file changed, 1 insertion(+), 1 deletion(-) - -Guy Korland in commit 7ded552d: - add missing argument to function doc - 1 file changed, 1 insertion(+), 1 deletion(-) +artix in commit 95324b81: + Support Redis Cluster Proxy PROXY INFO command + 1 file changed, 5 insertions(+), 1 deletion(-) -Pavel Skuratovich in commit f92b3273: - Fix typo in comment +박승현 in commit 04c53fa1: + Update redis.conf 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 57b6c343: - Modules: dictionary API WIP #10: export API to modules. - 2 files changed, 60 insertions(+) - -antirez in commit 3f82e59c: - Modules: dictionary API WIP #9: iterator returning string object. - 1 file changed, 23 insertions(+), 6 deletions(-) - -antirez in commit 6a73aca3: - Modules: dictionary API WIP #8: Iterator next/prev. - 1 file changed, 42 insertions(+) - -antirez in commit ef8413db: - Modules: dictionary API WIP #7: don't store the context. - 1 file changed, 7 insertions(+), 8 deletions(-) - -antirez in commit 05579e38: - Modules: dictionary API WIP #6: implement automatic memory management. - 1 file changed, 21 insertions(+), 7 deletions(-) - -antirez in commit 11c53f8c: - Modules: dictionary API work in progress #5: rename API for consistency. - 1 file changed, 25 insertions(+), 25 deletions(-) - -antirez in commit 0bd7091b: - Modules: change RedisModuleString API to allow NULL context. - 1 file changed, 33 insertions(+), 12 deletions(-) - -antirez in commit 5fc16f17: - Modules: dictionary API work in progress #4: reseek API. - 1 file changed, 25 insertions(+), 6 deletions(-) - -antirez in commit 45b7f779: - Modules: dictionary API work in progress #3: Iterator creation. - 1 file changed, 41 insertions(+), 1 deletion(-) - -antirez in commit 8576b0ae: - Modules: dictionary API work in progress #2: Del API. - 1 file changed, 17 insertions(+), 2 deletions(-) - -antirez in commit 4b0fa7a7: - Modules: dictionary API work in progress #1. - 2 files changed, 95 insertions(+), 1 deletion(-) - -antirez in commit 28210760: - Module cluster flags: use RM_SetClusterFlags() in the example. - 2 files changed, 11 insertions(+) - -antirez in commit 18c5ab93: - Module cluster flags: add RM_SetClusterFlags() API. - 3 files changed, 33 insertions(+) +WuYunlong in commit 0578157d: + Fix master replica inconsistency for upgrading scenario. + 3 files changed, 9 insertions(+), 2 deletions(-) -antirez in commit 4ce6bff2: - Module cluster flags: add hooks for NO_FAILOVER flag. - 1 file changed, 4 insertions(+), 2 deletions(-) - -antirez in commit 2ba52889: - Module cluster flags: add hooks for NO_REDIRECTION flag. - 3 files changed, 14 insertions(+), 4 deletions(-) - -antirez in commit 6a39ece6: - Module cluster flags: initial vars / defines added. - 5 files changed, 20 insertions(+) - -antirez in commit 0ff35370: - Modules: rename the reused static client to something more general. - 1 file changed, 10 insertions(+), 8 deletions(-) +WuYunlong in commit 299f1d02: + Add 14-consistency-check.tcl to prove there is a data consistency issue. + 1 file changed, 87 insertions(+) -antirez in commit 2d11ee95: - Modules: associate a fake client to timer context callback. - 1 file changed, 2 insertions(+) - -antirez in commit 851b2ed3: - Modules: associate a fake client to cluster message context callback. - 1 file changed, 2 insertions(+) +antirez in commit 61b98f32: + Regression test for #7011. + 1 file changed, 7 insertions(+) -artix in commit 148e4911: - Cluster Manager: clusterManagerFixOpenSlot now counts node's keys in slot if node is neither migrating nor importing. - 1 file changed, 20 insertions(+), 1 deletion(-) +antirez in commit 34ea2f4e: + ACL: default user off should not allow automatic authentication. + 2 files changed, 3 insertions(+), 2 deletions(-) -Guy Korland in commit 8afca145: - No need to return "OK" - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit cbbf9b39: + Sentinel: document auth-user directive. + 1 file changed, 12 insertions(+) -Guy Korland in commit 9a278db2: - typo fix - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit 9c2e42dd: + ACL: Make Redis 6 more backward compatible with requirepass. + 4 files changed, 17 insertions(+), 15 deletions(-) -antirez in commit 26479036: - Revert "fix repeat argument issue and reduce unnessary loop times for redis-cli." - 1 file changed, 7 insertions(+), 12 deletions(-) +antirez in commit d387f67d: + Sentinel: implement auth-user directive for ACLs. + 1 file changed, 38 insertions(+), 7 deletions(-) -Guy Korland in commit 27b7fb5a: - Fix few typos - 1 file changed, 10 insertions(+), 10 deletions(-) +zhaozhao.zz in commit 7c078416: + Threaded IO: bugfix client kill may crash redis + 1 file changed, 11 insertions(+), 5 deletions(-) -Guy Korland in commit 233aa2d3: - RedisModule_HashSet call must end with NULL +zhaozhao.zz in commit 9cc7038e: + Threaded IO: handle pending reads clients ASAP after event loop 1 file changed, 3 insertions(+), 1 deletion(-) -antirez in commit a8494072: - Sentinel: document how to undo a renamed command. - 1 file changed, 6 insertions(+), 1 deletion(-) - -antirez in commit 6c8a8f2e: - LOLWUT: split the command from version-specific implementations. - 3 files changed, 297 insertions(+), 241 deletions(-) - -antirez in commit 5c758406: - Slave removal: add a few forgotten aliases for CONFIG SET. - 1 file changed, 10 insertions(+) - -antirez in commit 2da823c4: - LOLWUT: add Redis version in the output. - 1 file changed, 3 insertions(+), 1 deletion(-) +antirez in commit da8c7c49: + Example sentinel conf: document requirepass. + 1 file changed, 8 insertions(+) -antirez in commit bfcba420: - LOLWUT: Ness -> Nees. - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit bdb338cf: + Aesthetic changes in PR #6989. + 1 file changed, 9 insertions(+), 5 deletions(-) -antirez in commit efed898a: - LOLWUT: Limit maximum CPU effort. +zhaozhao.zz in commit b3e03054: + Threaded IO: bugfix #6988 process events while blocked 1 file changed, 5 insertions(+) -antirez in commit eb0fbd71: - LOLWUT: change padding conditional to a more direct one. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Slobodan Mišković in commit ed08feb7: - Fix spelling descrive -> describe - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit e628f944: + Restore newline at the end of redis-cli.c + 1 file changed, 2 insertions(+), 1 deletion(-) -antirez in commit 2ffb4413: - LOLWUT: fix crash when col < 2. +chendianqiang in commit 5d4c4df3: + use correct list for moduleUnregisterUsedAPI 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 55dae693: - LOLWUT: fix structure typo in comment. +guodongxiaren in commit da14982d: + string literal should be const char* 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 9b3098b9: - LOLWUT: Fix license copyright year. - 1 file changed, 1 insertion(+), 1 deletion(-) +Itamar Haber in commit dc8885a1: + Adds keyspace notifications to migrate and restore + 1 file changed, 3 insertions(+), 1 deletion(-) -antirez in commit 263dbadc: - LOLWUT: increase the translation factor. +bodong.ybd in commit bfb18e55: + Remove duplicate obj files in Makefile 1 file changed, 2 insertions(+), 2 deletions(-) -antirez in commit a622f6c0: - LOLWUT: change default size to fit a normal terminal better. - 1 file changed, 6 insertions(+), 6 deletions(-) - -antirez in commit 38b0d25a: - LOLWUT: wrap it into a proper command. - 4 files changed, 40 insertions(+), 15 deletions(-) - -antirez in commit 34ebd898: - LOLWUT: draw Schotter by Georg Nees. - 1 file changed, 47 insertions(+), 3 deletions(-) - -antirez in commit 46286e64: - LOLWUT: draw rotated squares using trivial trigonometry. - 1 file changed, 44 insertions(+) - -antirez in commit 2d4143fd: - LOLWUT: draw lines using Bresenham algorithm. - 1 file changed, 26 insertions(+), 2 deletions(-) - -antirez in commit 3546d9ce: - LOLWUT: Rendering of the virtual canvas to a string. - 1 file changed, 78 insertions(+), 7 deletions(-) - -antirez in commit b404a6ce: - LOLWUT: show the output verbatim in redis-cli. - 1 file changed, 1 insertion(+) - -antirez in commit e30ba94f: - LOLWUT: canvas structure and BSD license on top. - 1 file changed, 46 insertions(+) - -antirez in commit 9c771145: - LOLWUT: Emit Braille unicode according to pixel pattern. - 1 file changed, 23 insertions(+) +bodong.ybd in commit 76d57161: + Fix bug of tcl test using external server + 2 files changed, 8 insertions(+), 2 deletions(-) -Jakub Vrana in commit 4a1d6c7d: - Slave removal: capitalize Replica - 2 files changed, 5 insertions(+), 5 deletions(-) +fengpf in commit 0e5820d8: + fix comments in latency.c + 2 files changed, 2 insertions(+), 1 deletion(-) -antirez in commit 72e0368a: - Slave removal: remove slave from integration tests descriptions. - 8 files changed, 36 insertions(+), 36 deletions(-) +antirez in commit 916dd79f: + Update linenoise. + 1 file changed, 2 insertions(+), 1 deletion(-) -antirez in commit c7841c2b: - Slave removal: remove slave from top-level tests descriptions. - 3 files changed, 12 insertions(+), 12 deletions(-) +lifubang in commit c0c67c9b: + add askpass mode + 1 file changed, 19 insertions(+), 1 deletion(-) -antirez in commit 1b9b19ba: - Slave removal: remove slave from object.c. - 1 file changed, 1 insertion(+), 1 deletion(-) +lifubang in commit e1c29434: + update linenoise to https://github.com/antirez/linenoise/tree/fc9667a81d43911a6690fb1e68c16e6e3bb8df05 + 4 files changed, 59 insertions(+), 4 deletions(-) -antirez in commit 7da266e6: - Slave removal: remove slave from the README. - 1 file changed, 7 insertions(+), 7 deletions(-) +Jamie Scott in commit e5a063bc: + Remove default guidance in Redis.conf + 1 file changed, 1 insertion(+), 2 deletions(-) -antirez in commit 93d803c9: - Slave removal: server.c logs fixed. - 1 file changed, 5 insertions(+), 5 deletions(-) +Jamie Scott in commit d28cbaf7: + Update Redis.conf to improve TLS usability + 1 file changed, 2 insertions(+), 1 deletion(-) -antirez in commit 89434032: - Slave removal: remove slave from sentinel.conf when possible. - 1 file changed, 18 insertions(+), 18 deletions(-) +Johannes Truschnigg in commit 23d5e8b8: + Signal systemd readiness atfer Partial Resync + 1 file changed, 4 insertions(+) -antirez in commit 7673d88d: - Slave removal: replace very few things in Sentinel. - 1 file changed, 12 insertions(+), 8 deletions(-) +Oran Agra in commit 61738154: + fix for flaky psync2 test + 1 file changed, 21 insertions(+) -antirez in commit f1de29b3: - Slave removal: scripting.c logs and other stuff fixed. +antirez in commit 70e0e499: + ae.c: fix crash when resizing the event loop. 1 file changed, 6 insertions(+), 2 deletions(-) -antirez in commit 53fe558e: - Slave removal: replication.c logs fixed. - 1 file changed, 35 insertions(+), 35 deletions(-) - -antirez in commit c92b02dd: - Slave removal: networking.c logs fixed. - 1 file changed, 5 insertions(+), 5 deletions(-) - -antirez in commit be76ed0c: - Slave removal: blocked.c logs fixed. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 3fd73151: - Slave removal: Make obvious in redis.conf what a replica is. - 1 file changed, 5 insertions(+) - -antirez in commit a22168e4: - Slave removal: slave mode -> replica mode text in redis-cli. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 0e222fbe: - Slave removal: fix typo of replicaof. +antirez in commit b3e4aa67: + Fix release notes spelling mistake. 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 34a5615e: - Slave removal: slave -> replica in redis.conf and output buffer option. - 3 files changed, 132 insertions(+), 129 deletions(-) -antirez in commit 1d2fcf6f: - Slave removal: Convert cluster.c log messages and command names. - 1 file changed, 12 insertions(+), 11 deletions(-) +================================================================================ +Redis 6.0 RC2 Released Thu Mar 05 15:40:53 CET 2020 +================================================================================ -antirez in commit 2546158d: - Slave removal: config.c converted + config rewriting hacks. - 1 file changed, 117 insertions(+), 38 deletions(-) +Upgrade urgency MODERATE: Normal bugfixing release of a non-GA branch. -antirez in commit c0952c0d: - Slave removal: redis-cli --slave -> --replica. - 1 file changed, 3 insertions(+), 1 deletion(-) +Hi Redis users, Redis 6 is approaching and will be released 30th of April. +New release candidates will be released at the end of March, then another +one mid April, to finally reach the GA at the end of April. -antirez in commit 1f37f1dd: - Slave removal: SLAVEOF -> REPLICAOF. SLAVEOF is now an alias. - 3 files changed, 4 insertions(+), 3 deletions(-) +Redis 6 RC2 brings many fixes and new things, especially in the area of +client side caching. This is the list of big changes in this release. As +usually you can find the full list of commits at the end: -Amin Mesbah in commit 7928f578: - Use geohash limit defines in constraint check - 1 file changed, 2 insertions(+), 2 deletions(-) +New features and improvements: -Jeffrey Lovitz in commit bb2bed78: - CLI Help text loop verifies arg count - 1 file changed, 1 insertion(+), 1 deletion(-) +* ACL LOG: log denied commands, keys accesses and authentications. +* Client side caching redesigned. Now we use keys not caching slots. +* Client side caching: Broadcasting mode implemented. +* Client side caching: OPTIN/OPTOUT modes implemented. +* Remove RDB files used for replication in persistence-less instances (option). -youjiali1995 in commit 246980d0: - sentinel: fix randomized sentinelTimer. - 1 file changed, 1 insertion(+), 3 deletions(-) +Fixes (only selected ones, see commits for all the fixes): -youjiali1995 in commit fa7de8c4: - bio: fix bioWaitStepOfType. - 1 file changed, 3 insertions(+), 3 deletions(-) +* Different fixes to streams in edge cases. +* Fix duplicated CLIENT SETNAME reply because of RESP3 changes. +* Fix crash due to new active expire division by zero. +* Avoid sentinel changes promoted_slave to be its own replica. +* Fix bug on KEYS command where pattern starts with * followed by \x00. +* Threaded I/O: now the main thread is used as well to do I/O. +* Many fixes to modules APIs, and more to come in the next RCs. +* ld2string should fail if string contains \0 in the middle. +* Make the Redis test more reliable. +* Fix SPOP returning nil (see #4709). WARNING: API change. -Weiliang Li in commit 7642f9d5: - fix usage typo in redis-cli +qetu3790 in commit 4af0d7fd: + Fix not used constant in lru_test_mode. 1 file changed, 1 insertion(+), 1 deletion(-) -================================================================================ -Redis 5.0 RC5 Released Thu Sep 06 12:54:29 CEST 2018 -================================================================================ +hwware in commit 6ef01878: + add missing file marco + 1 file changed, 5 insertions(+) -Upgrade urgency HIGH: Several imporant bugs fixed. - -Hi all, - -This is the release candidate number five, and has a lot of bug fixes inside, -together with a few big changes to the Redis behavior from the point of view -of replication of scripts and handling of the maxmemory directive in slaves. -Make sure to read the whole list! - -* BREAKING BEHAVIOR: Slaves now ignore maxmemory by default. -* BREAKING BEHAVIOR: Now scripts are always replicated for their effects, and - never sending the script itself to slaves/AOF. -* Improvement: Big pipelining performances improved significantly. -* Fix: Rewrite BRPOPLPUSH as RPOPLPUSH to propagate. -* Fix: False positives in tests. -* Fix: Certain command flags were modified because not correct. -* Fix: Fix blocking XREAD for streams that are empty. -* Improvement: Allow scripts to timeout on slaves as well. -* Fix: Different corner cases due to CLIENT PAUSE are now fixed. -* Improvement: Optimize parsing large bulk greater than 32k. -* Fix: Propagate read-only scripts as SCRIPT LOAD, not as EVAL. - -The following is the list of commits, so that you can read the details and -check the credits of the commits. - -antirez in commit 1d1bf7f0: - Document that effects replication is Redis 5 default. - 1 file changed, 8 insertions(+) +ShooterIT in commit fe81d5c8: + Avoid compiler warnings + 1 file changed, 1 insertion(+) -antirez in commit cfd969c7: - Fix scripting tests now that we default to commands repl. - 1 file changed, 8 insertions(+), 1 deletion(-) +antirez in commit c2f01d7f: + RDB deletion: document it in example redis.conf. + 1 file changed, 13 insertions(+) -antirez in commit 3e1fb5ff: - Use commands (effects) replication by default in scripts. - 3 files changed, 8 insertions(+), 1 deletion(-) +antirez in commit 127e09bc: + Make sync RDB deletion configurable. Default to no. + 3 files changed, 22 insertions(+), 4 deletions(-) -antirez in commit c6c71abe: - Safer script stop condition on OOM. - 1 file changed, 5 insertions(+), 2 deletions(-) +antirez in commit a20303c6: + Check that the file exists in removeRDBUsedToSyncReplicas(). + 1 file changed, 8 insertions(+), 4 deletions(-) -antirez in commit dfbce91a: - Propagate read-only scripts as SCRIPT LOAD. - 1 file changed, 16 insertions(+), 3 deletions(-) +antirez in commit 7a23b945: + Log RDB deletion in persistence-less instances. + 1 file changed, 15 insertions(+), 2 deletions(-) -antirez in commit 1705e42e: - Don't perform eviction when re-entering the event loop. - 1 file changed, 7 insertions(+), 2 deletions(-) +antirez in commit baaf869f: + Introduce bg_unlink(). + 1 file changed, 31 insertions(+), 3 deletions(-) -antirez in commit a0dd6f82: - Clarify why remaining may be zero in readQueryFromClient(). - 1 file changed, 2 insertions(+) +antirez in commit be4bc1a5: + Remove RDB files used for replication in persistence-less instances. + 3 files changed, 56 insertions(+), 1 deletion(-) -zhaozhao.zz in commit 2eed31a5: - networking: fix unexpected negative or zero readlen +antirez in commit 07dc1b42: + Use a smaller getkeys global buffer. 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 37fb606c: - Merge branch '5.0' of github.com:/antirez/redis into 5.0 -zhaozhao.zz in commit 1898e6ce: - networking: optimize parsing large bulk greater than 32k - 1 file changed, 13 insertions(+), 10 deletions(-) +Oran Agra in commit 10e71b3d: + Optimize temporary memory allocations for getKeysFromCommand mechanism + 1 file changed, 31 insertions(+), 10 deletions(-) -antirez in commit 82fc63d1: - Unblocked clients API refactoring. See #4418. - 4 files changed, 33 insertions(+), 15 deletions(-) +antirez in commit edc0ed14: + Modules: reformat RM_Scan() top comment a bit. + 1 file changed, 21 insertions(+), 12 deletions(-) -zhaozhao.zz in commit 839bb52c: - if master is already unblocked, do not unblock it twice - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit c5319612: + Modules: more details in RM_Scan API top comment. + 1 file changed, 22 insertions(+), 6 deletions(-) -zhaozhao.zz in commit 2e1cd82d: - fix multiple unblock for clientsArePaused() - 1 file changed, 3 insertions(+), 3 deletions(-) +Oran Agra in commit fff6b26a: + RM_Scan disable dict rehashing + 2 files changed, 21 insertions(+), 6 deletions(-) -antirez in commit 17233080: - Make pending buffer processing safe for CLIENT_MASTER client. - 3 files changed, 22 insertions(+), 13 deletions(-) +Guy Benoish in commit 65048460: + Add RM_CreateStringFromDouble + 2 files changed, 14 insertions(+) -antirez in commit 42bce87a: - Test: processing of master stream in slave -BUSY state. - 1 file changed, 44 insertions(+) - -antirez in commit 8bf42f60: - After slave Lua script leaves busy state, re-process the master buffer. - 2 files changed, 5 insertions(+), 2 deletions(-) - -antirez in commit c2b104c7: - While the slave is busy, just accumulate master input. - 2 files changed, 6 insertions(+), 1 deletion(-) +Oran Agra in commit 3144a278: + add no_auth to COMMAND INFO + 1 file changed, 1 insertion(+) -antirez in commit 7b75f4ae: - Allow scripts to timeout even if from the master instance. - 1 file changed, 6 insertions(+), 11 deletions(-) +Oran Agra in commit afe0b16c: + module api docs for aux_save and aux_load + 2 files changed, 7 insertions(+), 1 deletion(-) -antirez in commit adc4e031: - Allow scripts to timeout on slaves as well. - 2 files changed, 10 insertions(+), 3 deletions(-) +Guy Benoish in commit df152b0c: + streamReplyWithRangeFromConsumerPEL: Redundant streamDecodeID + 1 file changed, 1 insertion(+), 3 deletions(-) -dejun.xdj in commit 20ec1f0c: - Revise the comments of latency command. +antirez in commit e3c1f439: + Show Redis version when not understanding a config directive. 1 file changed, 2 insertions(+), 1 deletion(-) -Chris Lamb in commit 8e5423eb: - Correct "did not received" -> "did not receive" typos/grammar. - 6 files changed, 10 insertions(+), 10 deletions(-) - -zhaozhao.zz in commit 395063d7: - remove duplicate bind in sentinel.conf - 1 file changed, 10 deletions(-) - -Salvatore Sanfilippo in commit b221ca41: - Merge pull request #5300 from SaschaRoland/xread-block-5299 -Sascha Roland in commit eea0d3c5: - #5299 Fix blocking XREAD for streams that ran dry - 1 file changed, 2 insertions(+), 2 deletions(-) +antirez in commit 141c0679: + Changelog: explain Redis 6 SPOP change. + 1 file changed, 4 insertions(+), 1 deletion(-) -antirez in commit 4cb9ee11: - Add maxmemory slave behavior change in the change log. - 1 file changed, 8 insertions(+) +bodong.ybd in commit fe902461: + Fix spop return nil #4709 + 1 file changed, 1 insertion(+), 1 deletion(-) -zhaozhao.zz in commit 5ad888ba: - Supplement to PR #4835, just take info/memory/command as random commands +antirez in commit 9d4219eb: + Fix SDS misuse in enumConfigSet(). Related to #6778. 1 file changed, 3 insertions(+), 3 deletions(-) -zhaozhao.zz in commit d928487f: - some commands' flags should be set correctly, issue #4834 - 1 file changed, 14 insertions(+), 14 deletions(-) - -Oran Agra in commit af675f0a: - Fix unstable tests on slow machines. - 3 files changed, 23 insertions(+), 17 deletions(-) - -antirez in commit f2cd16be: - Document slave-ignore-maxmemory in redis.conf. - 1 file changed, 20 insertions(+) +antirez in commit 84243064: + Remove useless comment from enumConfigSet(). + 1 file changed, 1 deletion(-) -antirez in commit 02d729b4: - Make slave-ignore-maxmemory configurable. - 1 file changed, 9 insertions(+) +Ponnuvel Palaniyappan in commit dafb94db: + Fix a potential overflow with strncpy + 1 file changed, 5 insertions(+), 5 deletions(-) -antirez in commit 447da44d: - Introduce repl_slave_ignore_maxmemory flag internally. - 3 files changed, 7 insertions(+) +antirez in commit ea697b63: + Improve aeDeleteEventLoop() top comment grammar. + 1 file changed, 2 insertions(+), 1 deletion(-) -antirez in commit 868b2925: - Better variable meaning in processCommand(). - 1 file changed, 2 insertions(+), 2 deletions(-) +wangyuan21 in commit dd479880: + free time event when delete eventloop + 1 file changed, 7 insertions(+) -antirez in commit 319f2ee6: - Re-apply rebased #2358. - 1 file changed, 1 insertion(+), 1 deletion(-) +srzhao in commit ecf3b2ef: + fix impl of aof-child whitelist SIGUSR1 feature. + 1 file changed, 5 insertions(+), 4 deletions(-) -zhaozhao.zz in commit 22c166da: - block: format code +meir@redislabs.com in commit 2966132c: + Changed log level for module fork api from 'notice' to 'verbos'. 1 file changed, 2 insertions(+), 2 deletions(-) -zhaozhao.zz in commit c03c5913: - block: rewrite BRPOPLPUSH as RPOPLPUSH to propagate - 3 files changed, 5 insertions(+), 1 deletion(-) - -zhaozhao.zz in commit fcd5ef16: - networking: make setProtocolError simple and clear - 1 file changed, 11 insertions(+), 13 deletions(-) - -zhaozhao.zz in commit 656e4b2f: - networking: just move qb_pos instead of sdsrange in processInlineBuffer - 1 file changed, 2 insertions(+), 3 deletions(-) - -zhaozhao.zz in commit 2c7972ce: - networking: just return C_OK if multibulk processing saw a <= 0 length. - 1 file changed, 2 insertions(+), 5 deletions(-) - -zhaozhao.zz in commit 1203a04f: - adjust qbuf to 26 in test case for client list +hwware in commit 7277e5d8: + format fix 1 file changed, 1 insertion(+), 1 deletion(-) -zhaozhao.zz in commit aff86fa1: - pipeline: do not sdsrange querybuf unless all commands processed - 2 files changed, 48 insertions(+), 40 deletions(-) - -Chris Lamb in commit 45a6c5be: - Use SOURCE_DATE_EPOCH over unreproducible uname + date calls. - 1 file changed, 3 insertions(+) - -Chris Lamb in commit 186df148: - Make some defaults explicit in the sentinel.conf for package maintainers - 1 file changed, 25 insertions(+) +hwware in commit 1bb5ee9c: + fix potentical memory leaks + 1 file changed, 4 insertions(+), 1 deletion(-) -dejun.xdj in commit b59f04a0: - Streams: ID of xclaim command starts from the sixth argument. +Hengjian Tang in commit 97329733: + modify the read buf size according to the write buf size PROTO_IOBUF_LEN defined before 1 file changed, 1 insertion(+), 1 deletion(-) -shenlongxing in commit a3f2437b: - Fix stream command paras - 2 files changed, 7 insertions(+), 7 deletions(-) - -antirez in commit df911235: - Fix AOF comment to report the current behavior. - 1 file changed, 3 insertions(+), 1 deletion(-) - - - -================================================================================ -Redis 5.0 RC4 Released Fri Aug 03 13:51:02 CEST 2018 -================================================================================ - -Upgrade urgency - - HIGH: Many non critical but important issues fixed. - CRITICAL for Stream users: Many important bugs fixed. - -Hi all, welcome to Redis 5.0 RC4. - -This release is a huge step forward in Redis 5 maturity and fixes a number -of issues. It also provides interesting improvements. Here I'll summarize -the biggest one, but laster you can find the full list of commits: - -Fixes: - -* A number of fixes related to Streams: stability and correctnes. -* Fix dbRandomKey() potential infinite loop. -* Improve eviction LFU/LRU when keys are created by INCR commands family. -* Active defragmentation is now working on Redis 5. -* Fix corner case in Redis CLuster / Sentinel failover, by resetting the - disconnection time with master in a more appropriate place. -* Use a private version of localtime() to avoid potential deadlocks. -* Different redis-cli non critical fixes. -* Fix rare replication stream corruption with disk-based replication. - -Improvements: - -* Sentinel: add an option to deny online script reconfiguration. -* Improved RESTORE command. -* Sentinel command renaming: allows to use Sentinel with Redis instances - that have non standard command names. -* CLIENT ID and CLIENT UNBLOCK. -* CLIENT LIST now supports a TYPE option. -* redis-cli --cluster now supports authentication. -* redis-trib is now deprecated (use redis-cli --cluster). -* Better slaves output buffers efficiency. -* Faster INFO when there are many clients connected. -* Dynamic HZ feature. -* Improvements in what the MEMORY command is able to report. -* Add year in log. (WARNING: may be incompatible with log scraping tools) -* Lazy freeing now works even when values are overwritten (for instance SET). -* Faster ZADD when elements scores are updated. -* Improvements to the test suite, including many new options. - -antirez in commit a4d1201e: - Test suite: add --loop option. - 1 file changed, 12 insertions(+), 5 deletions(-) - -antirez in commit 273d8191: - Test suite: new --stop option. - 1 file changed, 13 insertions(+), 4 deletions(-) - -antirez in commit fbbcc6a6: - Streams IDs parsing refactoring. - 1 file changed, 32 insertions(+), 17 deletions(-) - -antirez in commit 70c4bcb7: - Test: new sorted set skiplist order consistency. - 1 file changed, 26 insertions(+) - -antirez in commit 63addc5c: - Fix zslUpdateScore() edge case. +Ariel in commit 15ea1324: + fix ThreadSafeContext lock/unlock function names 1 file changed, 2 insertions(+), 2 deletions(-) -antirez in commit 724740cc: - More commenting of zslUpdateScore(). - 1 file changed, 2 insertions(+) - -antirez in commit ddc87eef: - Explain what's the point of zslUpdateScore() in top comment. - 1 file changed, 5 insertions(+) - -antirez in commit 741f29ea: - Remove old commented zslUpdateScore() from source. - 1 file changed, 13 deletions(-) - -antirez in commit 20116836: - Optimize zslUpdateScore() as asked in #5179. - 1 file changed, 44 insertions(+) - -antirez in commit 8c297e8b: - zsetAdd() refactored adding zslUpdateScore(). - 1 file changed, 18 insertions(+), 7 deletions(-) - -dejun.xdj in commit bd2f3f6b: - Streams: rearrange the usage of '-' and '+' IDs in stream commands. - 1 file changed, 13 insertions(+), 13 deletions(-) - -dejun.xdj in commit c0c06b84: - Streams: add mmid_supp argument in streamParseIDOrReply(). - 1 file changed, 6 insertions(+), 2 deletions(-) - -antirez in commit ab237a8e: - Minor improvements to PR #5187. - 2 files changed, 13 insertions(+), 6 deletions(-) - -Oran Agra in commit 1ce3cf7a: - test suite conveniency improvements - 3 files changed, 79 insertions(+), 3 deletions(-) - -Oran Agra in commit 36622899: - add DEBUG LOG, to to assist test suite debugging - 1 file changed, 4 insertions(+) - -antirez in commit 83d4311a: - Cluster cron announce IP minor refactoring. - 1 file changed, 6 insertions(+), 3 deletions(-) - -shenlongxing in commit a633f8e1: - Fix cluster-announce-ip memory leak - 1 file changed, 3 insertions(+), 2 deletions(-) - -antirez in commit 24c45538: - Tranfer -> transfer typo fixed. - 1 file changed, 1 insertion(+), 1 deletion(-) - -zhaozhao.zz in commit c609f240: - refactor dbOverwrite to make lazyfree work - 4 files changed, 27 insertions(+), 12 deletions(-) - -antirez in commit 9e971739: - Refactoring: replace low-level checks with writeCommandsDeniedByDiskError(). - 2 files changed, 6 insertions(+), 13 deletions(-) - -antirez in commit 0e77cef0: - Fix writeCommandsDeniedByDiskError() inverted return value. +Guy Benoish in commit 4d12c37c: + XREADGROUP should propagate XCALIM/SETID in MULTI/EXEC 1 file changed, 2 insertions(+), 2 deletions(-) -antirez in commit acfe9d13: - Better top comment for writeCommandsDeniedByDiskError(). - 1 file changed, 8 insertions(+), 1 deletion(-) - -antirez in commit 4e933e00: - Introduce writeCommandsDeniedByDiskError(). - 2 files changed, 24 insertions(+) - -WuYunlong in commit 41607dfd: - Consider aof write error as well as rdb in lua script. - 1 file changed, 14 insertions(+), 4 deletions(-) - -Salvatore Sanfilippo in commit 1d073a64: - Merge pull request #5168 from rpv-tomsk/issue-5033 -Guy Korland in commit 2db31fd4: - Few typo fixes - 1 file changed, 13 insertions(+), 13 deletions(-) - -antirez in commit 64242757: - Add year in log. - 1 file changed, 1 insertion(+), 1 deletion(-) +Oran Agra in commit 12626ce9: + fix race in module api test for fork + 2 files changed, 2 insertions(+), 3 deletions(-) -antirez in commit 50be4a1f: - Document dynamic-hz in the example redis.conf. - 1 file changed, 16 insertions(+) +Guy Benoish in commit 2ecab0b6: + Modules: Do not auto-unblock clients if not blocked on keys + 1 file changed, 22 insertions(+), 7 deletions(-) -antirez in commit 9a76472d: - Make dynamic hz actually configurable. - 1 file changed, 9 insertions(+) +Oran Agra in commit 635321d4: + fix github actions failing latency test for active defrag - part 2 + 2 files changed, 5 insertions(+), 4 deletions(-) -antirez in commit a330d06c: - Control dynamic HZ via server configuration. - 2 files changed, 13 insertions(+), 6 deletions(-) +Oran Agra in commit 0b988fa9: + fix github actions failing latency test for active defrag + 2 files changed, 14 insertions(+), 13 deletions(-) -antirez in commit d42602ff: - Dynamic HZ: adapt cron frequency to number of clients. - 2 files changed, 17 insertions(+), 5 deletions(-) +Oran Agra in commit 60096bc1: + Fix latency sensitivity of new defrag test + 1 file changed, 32 insertions(+), 8 deletions(-) -antirez in commit 7b5f0223: - Dynamic HZ: separate hz from the configured hz. - 3 files changed, 15 insertions(+), 9 deletions(-) +antirez in commit b4395426: + Tracking: optin/out implemented. + 3 files changed, 82 insertions(+), 16 deletions(-) -antirez in commit 037b00de: - Remove useless conditional from emptyDb(). - 1 file changed, 1 deletion(-) +antirez in commit ef3551d1: + Test engine: experimental change to avoid busy port problems. + 1 file changed, 84 insertions(+), 49 deletions(-) -antirez in commit 0e97ae79: - Make emptyDb() change introduced in #4852 simpler to read. - 1 file changed, 8 insertions(+), 3 deletions(-) +antirez in commit 72c05351: + Test engine: detect timeout when checking for Redis startup. + 1 file changed, 11 insertions(+), 1 deletion(-) -zhaozhao.zz in commit f7740faf: - optimize flushdb, avoid useless loops - 1 file changed, 5 insertions(+), 2 deletions(-) +antirez in commit 294c9af4: + Test engine: better tracking of what workers are doing. + 2 files changed, 12 insertions(+), 4 deletions(-) -zhaozhao.zz in commit 0c008376: - Streams: fix xdel memory leak +hwware in commit ba027079: + add missing subcommand description for debug oom 1 file changed, 1 insertion(+) -antirez in commit dc600a25: - Example the magic +1 in migrateCommand(). +Guy Benoish in commit 5d0890c0: + Fix memory leak in test_ld_conv 1 file changed, 4 insertions(+) -antirez in commit d6827ab6: - Make changes of PR #5154 hopefully simpler. - 1 file changed, 10 insertions(+), 5 deletions(-) - -WuYunlong in commit 89ec1453: - Do not migrate already expired keys. - 1 file changed, 6 insertions(+), 2 deletions(-) - -Pavel Rochnyack in commit cd25ed17: - INFO CPU: higher precision of reported values - 1 file changed, 8 insertions(+), 8 deletions(-) - -antirez in commit 6bfb4745: - Streams: refactoring of next entry seek in the iterator. - 1 file changed, 11 insertions(+), 7 deletions(-) - -zhaozhao.zz in commit 4724548e: - Streams: skip master fileds only when we are going forward in streamIteratorGetID - 1 file changed, 8 insertions(+), 5 deletions(-) - -Oran Agra in commit 4b79fdf1: - fix slave buffer test suite false positives - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit a1e081f7: - string2ll(): better commenting. - 1 file changed, 6 insertions(+) - -dsomeshwar in commit 8b4fe752: - removing redundant check - 1 file changed, 3 deletions(-) - -antirez in commit 9e5bf047: - Restore string2ll() to original version. - 1 file changed, 7 insertions(+), 2 deletions(-) - -Oran Agra in commit c2ecdcde: - fix recursion typo in zmalloc_usable - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 4f742bd6: - string2ll(): remove duplicated check for special case. - 1 file changed, 1 insertion(+), 6 deletions(-) - -antirez in commit a4efac00: - string2ll(): test for NULL pointer in all the cases. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 2c07c107: - Change 42 to 1000 as warning level for cached scripts. - 1 file changed, 3 insertions(+), 3 deletions(-) - -Itamar Haber in commit 270903d6: - Adds Lua overheads to MEMORY STATS, smartens the MEMORY DOCTOR - 3 files changed, 30 insertions(+), 4 deletions(-) - -Itamar Haber in commit faf3dbfc: - Adds memory information about the script's cache to INFO - 3 files changed, 12 insertions(+) - -antirez in commit 49841a54: - Fix merge errors. - 2 files changed, 7 deletions(-) - -antirez in commit 77a7ec72: - Merge branch 'unstable' into 5.0 branch -antirez in commit 4ff47a0b: - Top comment clientsCron(). - 1 file changed, 19 insertions(+), 4 deletions(-) - -antirez in commit aba68552: - Clarify that clientsCronTrackExpansiveClients() indexes may jump ahead. - 1 file changed, 9 insertions(+), 1 deletion(-) - -antirez in commit be88c0b1: - Rename INFO CLIENT max buffers field names for correctness. - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit 0cf3794e: - Fix wrong array index variable in getExpansiveClientsInfo(). - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit ea3a20c5: - Change INFO CLIENTS sections to report pre-computed max/min client buffers. - 1 file changed, 5 insertions(+), 5 deletions(-) - -antirez in commit 8f7e496b: - Rename var in clientsCronTrackExpansiveClients() for clarity. - 1 file changed, 3 insertions(+), 3 deletions(-) - -antirez in commit 8d617596: - Implement a function to retrieve the expansive clients mem usage. - 1 file changed, 12 insertions(+) - -antirez in commit 85a1b4f8: - clientsCronTrackExpansiveClients() actual implementation. - 1 file changed, 14 insertions(+), 1 deletion(-) - -antirez in commit d4c5fc57: - clientsCronTrackExpansiveClients() skeleton and ideas. - 1 file changed, 23 insertions(+) - -antirez in commit 1c95c075: - Make vars used only by INFO CLIENTS local to the block. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Salvatore Sanfilippo in commit 16b8d364: - Merge pull request #4727 from kingpeterpaule/redis-fix-info-cli -antirez in commit 0aca977c: - Merge branch 'unstable' of github.com:/antirez/redis into unstable -antirez in commit 313b2240: - In addReplyErrorLength() only panic when replying to slave. - 1 file changed, 4 insertions(+), 3 deletions(-) - -antirez in commit 6183f059: - Refine comment in addReplyErrorLength() about replying to masters/slaves. - 1 file changed, 11 insertions(+) - -Salvatore Sanfilippo in commit 22e9321c: - Merge pull request #5138 from oranagra/improve_defrag_test -Oran Agra in commit f89c93c8: - make active defrag test more stable - 2 files changed, 6 insertions(+), 5 deletions(-) - -Salvatore Sanfilippo in commit 8213f64d: - Merge pull request #5122 from trevor211/allowWritesWhenAofDisabled -Salvatore Sanfilippo in commit 46fd9278: - Merge pull request #4237 from aspirewit/update-comment -antirez in commit 6201f7b4: - Streams: better error when $ is given with XREADGROUP. - 1 file changed, 5 insertions(+), 2 deletions(-) - -Salvatore Sanfilippo in commit 4bff45c7: - Merge pull request #5136 from 0xtonyxia/fix-xread-id-parse -antirez in commit afc7e08a: - Panic when we are sending an error to our master/slave. - 1 file changed, 5 insertions(+) +Madelyn Olson in commit d1f22eac: + Give an error message if you specify redirect twice + 1 file changed, 7 insertions(+) -Salvatore Sanfilippo in commit e03358c0: - Merge pull request #5135 from oranagra/rare_repl_corruption -dejun.xdj in commit 846cf12a: - Streams: remove meaningless if condition. - 1 file changed, 1 insertion(+), 1 deletion(-) +Madelyn Olson in commit 762fbcb6: + Minor CSC fixes and fixed documentation + 2 files changed, 16 insertions(+), 17 deletions(-) -dejun.xdj in commit 6501b6bb: - Streams: return an error message if using xreadgroup with '$' ID. - 1 file changed, 5 insertions(+) +Oran Agra in commit 349aa245: + Defrag big lists in portions to avoid latency and freeze + 4 files changed, 350 insertions(+), 34 deletions(-) -Oran Agra in commit d5559898: - fix rare replication stream corruption with disk-based replication - 3 files changed, 18 insertions(+), 9 deletions(-) +Guy Benoish in commit b4ddc7b7: + XGROUP DESTROY should unblock XREADGROUP with -NOGROUP + 2 files changed, 11 insertions(+) -antirez in commit cefe21d2: - dict.c: remove a few trailing spaces. +hayashier in commit 73806f74: + fix typo from fss to rss 1 file changed, 2 insertions(+), 2 deletions(-) -Salvatore Sanfilippo in commit 4fc20992: - Merge pull request #5128 from kingpeterpaule/remove-one-loop-in-freeMemoryIfNeeded -Salvatore Sanfilippo in commit 9fbd49bb: - Merge pull request #5113 from 0xtonyxia/using-compare-func-instead -Salvatore Sanfilippo in commit cab39676: - Merge pull request #5127 from oranagra/sds_req_type -antirez in commit f9c84d6d: - Hopefully improve commenting of #5126. - 2 files changed, 22 insertions(+), 10 deletions(-) - -Salvatore Sanfilippo in commit e22a1218: - Merge pull request #5126 from oranagra/slave_buf_memory_2 -Salvatore Sanfilippo in commit 28dd8dd1: - Merge pull request #5132 from soloestoy/propagate-xdel-correctly -Oran Agra in commit bf680b6f: - slave buffers were wasteful and incorrectly counted causing eviction - 10 files changed, 182 insertions(+), 50 deletions(-) - -zhaozhao.zz in commit 73306c6f: - Streams: correctly propagate xdel if needed - 1 file changed, 7 insertions(+), 3 deletions(-) - -antirez in commit 103c5a1a: - Add a few comments to streamIteratorRemoveEntry(). - 1 file changed, 4 insertions(+) - -Salvatore Sanfilippo in commit a317f55d: - Merge pull request #5131 from soloestoy/optimize-xdel -antirez in commit 185e0d9c: - Modify XINFO field from last-id to last-generated-id. +antirez in commit b6129f86: + Test is more complex now, increase default timeout. 1 file changed, 1 insertion(+), 1 deletion(-) -Salvatore Sanfilippo in commit 4215e74b: - Merge pull request #5129 from soloestoy/xinfo-show-last-id -zhaozhao.zz in commit c9324f81: - Streams: free lp if all elements are deleted - 1 file changed, 9 insertions(+), 4 deletions(-) - -paule in commit b6ce7d5d: - Update dict.c - 1 file changed, 4 insertions(+), 2 deletions(-) - -zhaozhao.zz in commit b4ba5ac8: - Streams: show last id for streams and groups - 1 file changed, 6 insertions(+), 2 deletions(-) - -peterpaule in commit 816fc6cb: - remove one ineffective loop in dictGetSomeKeys. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit b05a22e2: - bugfix in sdsReqType creating 64bit sds headers on 32bit systems - 1 file changed, 3 insertions(+), 1 deletion(-) - -dejun.xdj in commit 491682a6: - Streams: using streamCompareID() instead of direct compare in block.c. - 1 file changed, 1 insertion(+), 4 deletions(-) +antirez in commit f15fb727: + Tracking: fix max-keys configuration directive. + 2 files changed, 2 insertions(+), 2 deletions(-) -dejun.xdj in commit a2177cd2: - Streams: add streamCompareID() declaration in stream.h. +Itamar Haber in commit e374573f: + Fixes segfault on calling trackingGetTotalKeys 1 file changed, 1 insertion(+) -dejun.xdj in commit 0484dbcf: - Streams: using streamCompareID() instead of direct compare. - 1 file changed, 2 insertions(+), 6 deletions(-) - -WuYunlong in commit 2d4366c5: - Accept write commands if persisting is disabled, event if we do have problems persisting on disk previously. - 1 file changed, 2 insertions(+), 1 deletion(-) - -Salvatore Sanfilippo in commit ab33bcd3: - Merge pull request #5120 from andrewsensus/leap-year-comment-patch-1 -antirez in commit 2352a519: - Test: XDEL fuzz testing. Remove and check stage. - 1 file changed, 15 insertions(+) - -antirez in commit 3d7d20b7: - Test: fix lshuffle by providing the "K" combinator. - 1 file changed, 2 insertions(+) - -antirez in commit 967ad364: - Test: add lshuffle in the Tcl utility functions set. - 1 file changed, 14 insertions(+) +antirez in commit 73d47d57: + Signal key as modified when expired on-access. + 1 file changed, 4 insertions(+), 2 deletions(-) -antirez in commit d6efd5fc: - Test: XDEL fuzz testing, stream creation. - 1 file changed, 20 insertions(+) +antirez in commit b7cb28d5: + Tracking: first set of tests for the feature. + 1 file changed, 66 insertions(+) -andrewsensus in commit 8dc08ae2: - update leap year comment +antirez in commit 1db72571: + Tracking: fix operators precedence error in bcast check. 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 69997153: - Merge branch 'unstable' of github.com:/antirez/redis into unstable -antirez in commit a93f8f09: - Test: XDEL basic test. - 1 file changed, 12 insertions(+) - -Salvatore Sanfilippo in commit a44a5934: - Merge pull request #5119 from trevor211/fixSlowlogConfig -WuYunlong in commit d6ba4fd5: - Fix config set slowlog-log-slower-than and condition in createLatencyReport - 2 files changed, 2 insertions(+), 2 deletions(-) - -WuYunlong in commit b3660be8: - Add test in slowlog.tcl - 1 file changed, 10 insertions(+) - -artix in commit d4182a0a: - Cluster Manager: more checks on --cluster-weight option. - 1 file changed, 12 insertions(+), 2 deletions(-) - -artix in commit d222eda9: - Redis-trib deprecated: it no longer works and it outputs a warning to the user. - 1 file changed, 103 insertions(+), 1804 deletions(-) - -artix in commit 513eb572: - Cluster Manager: auth support (-a argument). - 1 file changed, 41 insertions(+), 19 deletions(-) - -Salvatore Sanfilippo in commit f3980bb9: - Merge pull request #5115 from shenlongxing/patch-1 -Shen Longxing in commit c2a85fb3: - Delete unused role checking. - 1 file changed, 2 insertions(+), 6 deletions(-) - -Salvatore Sanfilippo in commit 4cb5bd4e: - Merge pull request #4820 from charpty/wip-serverc-simplify -antirez in commit 8d6b7861: - Add regression test for #5111. - 1 file changed, 15 insertions(+) - -antirez in commit b6260a02: - Streams: when re-delivering because of SETID, reset deliveries counter. - 1 file changed, 2 insertions(+), 2 deletions(-) +antirez in commit fe96e29d: + Tracking: fix behavior when switchinig from normal to BCAST. + 1 file changed, 11 insertions(+), 1 deletion(-) -antirez in commit a7c180e5: - Simplify duplicated NACK #5112 fix. - 1 file changed, 18 insertions(+), 21 deletions(-) +antirez in commit f21be1ec: + Tracking: fix sending messages bug + tracking off bug. + 2 files changed, 28 insertions(+), 20 deletions(-) -Salvatore Sanfilippo in commit bf4def0f: - Merge pull request #5112 from 0xtonyxia/fix-xreadgroup-crash-after-setid -Salvatore Sanfilippo in commit 16770551: - Merge pull request #5114 from oranagra/defrag_32 -Oran Agra in commit 920158ec: - Active defrag fixes for 32bit builds (again) - 1 file changed, 2 insertions(+), 2 deletions(-) +antirez in commit 6fb1aa23: + Tracking: BCAST: basic feature now works. + 3 files changed, 55 insertions(+), 40 deletions(-) -Salvatore Sanfilippo in commit f45e7901: - Merge pull request #4967 from JingchengLi/unstable -tengfeng in commit 9505dd20: - fix repeat argument issue and reduce unnessary loop times for redis-cli. - 1 file changed, 12 insertions(+), 7 deletions(-) +antirez in commit d4fe79a1: + Tracking: BCAST: broadcasting of keys in prefixes implemented. + 2 files changed, 94 insertions(+), 9 deletions(-) -antirez in commit 0420c327: - Merge branch 'unstable' of github.com:/antirez/redis into unstable -antirez in commit 28e95c7c: - Streams: fix typo "consumer". - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit abb81c63: + Tracking: BCAST: registration in the prefix table. + 3 files changed, 67 insertions(+), 20 deletions(-) -antirez in commit a8c1bb31: - Streams: fix new XREADGROUP sync logic. - 1 file changed, 13 insertions(+), 3 deletions(-) +antirez in commit 77da9608: + Tracking: BCAST: parsing of the options + skeleton. + 4 files changed, 73 insertions(+), 19 deletions(-) -antirez in commit 1a02b5f6: - Streams: make blocking for > a truly special case. - 1 file changed, 29 insertions(+), 4 deletions(-) - -antirez in commit a71e8148: - Streams: send an error to consumers blocked on non-existing group. - 1 file changed, 5 insertions(+), 1 deletion(-) - -antirez in commit 09327f11: - Streams: fix unblocking logic into a consumer group. - 1 file changed, 24 insertions(+), 14 deletions(-) - -dejun.xdj in commit 3f8a3efe: - Streams: fix xreadgroup crash after xgroup SETID is sent. - 1 file changed, 20 insertions(+), 15 deletions(-) +antirez in commit 3e8c69a9: + Tracking: always reply with an array of keys. + 2 files changed, 10 insertions(+), 3 deletions(-) -Salvatore Sanfilippo in commit 7239e9ca: - Merge pull request #5095 from 0xtonyxia/fix-indentation -dejun.xdj in commit 61f12973: - Bugfix: PEL is incorrect when consumer is blocked using xreadgroup with NOACK option. - 4 files changed, 6 insertions(+), 1 deletion(-) +antirez in commit a788c373: + Tracking: minor change of names and new INFO field. + 4 files changed, 11 insertions(+), 4 deletions(-) -antirez in commit b67f0276: - redis-cli: fix #4990 additional argument in help. +antirez in commit df838927: + Rax.c: populate data field after random walk. 1 file changed, 1 insertion(+) -antirez in commit 18d65849: - redis-cli: fix #5096 double error message. +antirez in commit 0517da36: + Tracking: rename INFO field with total items. 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 84620327: - redis-cli: cliConnect() flags CC_FORCE and CC_QUIET. - 1 file changed, 23 insertions(+), 13 deletions(-) - -Amit Dey in commit a3a5a25f: - fixing broken link in CONTRIBUTING - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit 3c16d6b3: + Tracking: first conversion from hashing to key names. + 3 files changed, 84 insertions(+), 114 deletions(-) -dejun.xdj in commit 289d8d9c: - CLIENT UNBLOCK: fix client unblock help message. - 1 file changed, 1 insertion(+), 1 deletion(-) - -minkikim89 in commit 62a4a8c1: - fix whitespace in redis-cli.c - 1 file changed, 362 insertions(+), 362 deletions(-) - -WuYunlong in commit 0a5805d7: - fix compile warning in addReplySubcommandSyntaxError - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit cb78c842: - Use nolocks_localtime() for safer logging. - 2 files changed, 8 insertions(+), 2 deletions(-) - -antirez in commit 81778d91: - Cache timezone and daylight active flag for safer logging. - 2 files changed, 14 insertions(+), 1 deletion(-) - -antirez in commit 18d8205b: - Localtime: clarify is_leap_year() working with comments. - 1 file changed, 4 insertions(+), 4 deletions(-) - -antirez in commit 29644144: - Localtime: fix comment about leap year. - 1 file changed, 1 insertion(+), 1 deletion(-) +Oran Agra in commit 3b4f1477: + add no-slowlog option to RM_CreateCommand + 1 file changed, 3 insertions(+) -antirez in commit 0ea39aa4: - Localtime: fix daylight saving adjustment. Use * not +. +Khem Raj in commit 5e762d84: + Mark extern definition of SDS_NOINIT in sds.h 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit 6614d305: - Localtime: fix daylight time documentation and computation. - 1 file changed, 14 insertions(+), 4 deletions(-) - -dejun.xdj in commit 46f5a2ca: - Fix indentation. - 2 files changed, 14 insertions(+), 14 deletions(-) - -antirez in commit 243c5a7a: - Localtime: add a test main() function to check the output. - 1 file changed, 15 insertions(+), 1 deletion(-) - -antirez in commit c25ee35a: - Localtime: day of month is 1 based. Convert from 0 based "days". +lifubang in commit 54f5499a: + correct help info for --user and --pass 1 file changed, 2 insertions(+), 2 deletions(-) -antirez in commit b9f33830: - Localtime: fix timezone adjustment. - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 0c12cbed: - Localtime: compute year, month and day of the month. - 1 file changed, 26 insertions(+) - -antirez in commit 06ca400f: - Localtime: basics initial calculations. Year missing. - 1 file changed, 30 insertions(+), 2 deletions(-) +Seunghoon Woo in commit 0c952b13: + [FIX] revisit CVE-2015-8080 vulnerability + 1 file changed, 6 insertions(+), 4 deletions(-) -antirez in commit 6a529067: - Localtime function skeleton and file added. - 1 file changed, 44 insertions(+) +Guy Benoish in commit dd34f703: + Diskless-load emptyDb-related fixes + 3 files changed, 44 insertions(+), 28 deletions(-) -Jack Drogon in commit 93238575: - Fix typo - 40 files changed, 99 insertions(+), 99 deletions(-) +lifubang in commit 5e042dbc: + fix ssl flag check for redis-cli + 1 file changed, 10 insertions(+), 9 deletions(-) -antirez in commit 94b3ee61: - Clarify the pending_querybuf field of clients. - 1 file changed, 4 insertions(+), 3 deletions(-) +Guy Benoish in commit dcbe8bfa: + Exclude "keymiss" notification from NOTIFY_ALL + 5 files changed, 12 insertions(+), 7 deletions(-) -antirez in commit 549b8b99: - Improve style of PR #5084. - 1 file changed, 8 insertions(+), 2 deletions(-) - -Salvatore Sanfilippo in commit 526b30a7: - Merge pull request #5084 from chendq8/pending-querybuf -antirez in commit 677d10b2: - Set repl_down_since to zero on state change. +Oran Agra in commit 36caf2e4: + update RM_SignalModifiedKey doc comment 1 file changed, 2 insertions(+), 1 deletion(-) -Salvatore Sanfilippo in commit 02e38516: - Merge pull request #5081 from trevor211/fixClusterFailover -chendianqiang in commit cbb2ac07: - Merge branch 'unstable' into pending-querybuf -antirez in commit 2edcafb3: - addReplySubSyntaxError() renamed to addReplySubcommandSyntaxError(). - 12 files changed, 14 insertions(+), 14 deletions(-) - -Salvatore Sanfilippo in commit bc6a0045: - Merge pull request #4998 from itamarhaber/module_command_help -Salvatore Sanfilippo in commit ee09b5ed: - Merge pull request #5071 from akshaynagpal/patch-2 -Salvatore Sanfilippo in commit f03ad962: - Merge pull request #5068 from shenlongxing/fix-rename-command -Salvatore Sanfilippo in commit e4881cd0: - Merge pull request #5090 from trevor211/test_helper_tcl -WuYunlong in commit 2833cfbe: - fix tests/test_helper.tcl with --wait-server option. Issue #5063 added --wait-server option, but can not work. - 1 file changed, 1 deletion(-) +Oran Agra in commit 3067352a: + Add handling of short read of module id in rdb + 1 file changed, 4 insertions(+), 1 deletion(-) -chendianqiang in commit 7de1ada0: - limit the size of pending-querybuf in masterclient - 4 files changed, 48 insertions(+) +Yossi Gottlieb in commit 9baaf858: + TLS: Update documentation. + 2 files changed, 32 insertions(+), 31 deletions(-) -WuYunlong in commit 2e167f7d: - fix server.repl_down_since resetting, so that slaves could failover automatically as expected. - 1 file changed, 1 insertion(+), 1 deletion(-) +Oran Agra in commit 4440133e: + A few non-data commands that should be allowed while loading or stale + 1 file changed, 8 insertions(+), 8 deletions(-) -WuYunlong in commit aeb7bc3e: - cluster.tcl: Add master consecutively down test. - 1 file changed, 77 insertions(+) +Oran Agra in commit c9577941: + Memory leak when bind config is provided twice + 1 file changed, 4 insertions(+) -antirez in commit d751d98b: - Change CLIENT LIST TYPE help string. - 1 file changed, 2 insertions(+), 2 deletions(-) +Oran Agra in commit 1333a46b: + fix maxmemory config warning + 1 file changed, 3 insertions(+), 2 deletions(-) -Salvatore Sanfilippo in commit a0b05a04: - Merge pull request #5075 from soloestoy/client-list-types -Salvatore Sanfilippo in commit aa2c390e: - Merge pull request #5074 from soloestoy/fix-compile-warning -Salvatore Sanfilippo in commit a4ef94d2: - Merge pull request #5076 from 0xtonyxia/add-no-auth-warning-option -dejun.xdj in commit 9f185626: - Check if password is used on command line interface. - 1 file changed, 1 insertion(+), 1 deletion(-) +Oran Agra in commit 8e7282eb: + Fix client flags to be int64 in module.c + 1 file changed, 3 insertions(+), 3 deletions(-) -dejun.xdj in commit 1139070a: - Fix trailing white space. - 1 file changed, 1 insertion(+), 1 deletion(-) +Oran Agra in commit a678390e: + moduleRDBLoadError, add key name, and use panic rather than exit + 1 file changed, 5 insertions(+), 4 deletions(-) -dejun.xdj in commit bbd0ca95: - Fix code format issue. - 1 file changed, 4 insertions(+), 4 deletions(-) +Oran Agra in commit 919fbf42: + reduce repeated calls to use_diskless_load + 1 file changed, 3 insertions(+), 4 deletions(-) -dejun.xdj in commit 7becf54e: - Don't output password warning message when --no-auth-warning is used. - 1 file changed, 10 insertions(+), 1 deletion(-) +Oran Agra in commit 22e45d46: + freeClientAsync don't lock mutex if there's just one thread + 1 file changed, 6 insertions(+), 1 deletion(-) -dejun.xdj in commit bde05e9c: - Avoid -Woverlength-strings compile warning. - 1 file changed, 5 insertions(+), 3 deletions(-) +Oran Agra in commit ba289244: + move restartAOFAfterSYNC from replicaofCommand to replicationUnsetMaster + 1 file changed, 4 insertions(+), 3 deletions(-) -antirez in commit 5baf50d8: - Rax library updated (node callback). - 2 files changed, 19 insertions(+), 5 deletions(-) +Oran Agra in commit f42ce57d: + stopAppendOnly resets aof_rewrite_scheduled + 1 file changed, 1 insertion(+) -dejun.xdj in commit 0b74fd67: - Add --no-auth-warning help message. +Oran Agra in commit df096bc9: + add SAVE subcommand to ACL HELP and top comment 1 file changed, 2 insertions(+) -zhaozhao.zz in commit b9cbd04b: - clients: add type option for client list - 4 files changed, 20 insertions(+), 6 deletions(-) - -zhaozhao.zz in commit f5538642: - clients: show pubsub flag in client list - 1 file changed, 1 insertion(+) +Oran Agra in commit a55e5847: + DEBUG HELP - add PROTOCOL + 1 file changed, 3 insertions(+), 2 deletions(-) -zhaozhao.zz in commit 1fcf2737: - fix some compile warnings - 2 files changed, 2 insertions(+), 2 deletions(-) +Guy Benoish in commit 5a6cfbf4: + Some refactroing using getClientType instead of CLIENT_SLAVE + 2 files changed, 18 insertions(+), 26 deletions(-) -Akshay Nagpal in commit 007e3cbd: - Added link to Google Group - 1 file changed, 3 insertions(+), 1 deletion(-) +Guy Benoish in commit fae306b3: + Fix small bugs related to replica and monitor ambiguity + 2 files changed, 8 insertions(+), 6 deletions(-) -antirez in commit ab55f9da: - Make CLIENT HELP output nicer to the eyes. - 1 file changed, 11 insertions(+), 11 deletions(-) +Yossi Gottlieb in commit 73630966: + TLS: Some redis.conf clarifications. + 1 file changed, 10 insertions(+), 11 deletions(-) -antirez in commit 75f1a7bd: - Merge branch 'unstable' of github.com:/antirez/redis into unstable -antirez in commit 4a70ff74: - Add unblock in CLIENT HELP. - 1 file changed, 1 insertion(+) +Oran Agra in commit 488e1947: + config.c verbose error replies for CONFIG SET, like config file parsing + 1 file changed, 31 insertions(+), 97 deletions(-) -shenlongxing in commit 3c27db1c: - fix empty string for sentinel rename-command - 1 file changed, 5 insertions(+) +Oran Agra in commit c82ccf06: + memoryGetKeys helper function so that ACL can limit access to keys for MEMORY command + 3 files changed, 18 insertions(+), 1 deletion(-) -Salvatore Sanfilippo in commit f7b21bc7: - Merge pull request #5066 from oranagra/defrag_jemalloc5_fix -Salvatore Sanfilippo in commit 730a4cfa: - Merge pull request #5067 from mpaltun/mpaltun-doc-fix -antirez in commit 2214043b: - CLIENT UNBLOCK: support unblocking by error. - 1 file changed, 22 insertions(+), 3 deletions(-) +antirez in commit 51c1a9f8: + ACL LOG: make max log entries configurable. + 4 files changed, 19 insertions(+) -Mustafa Paltun in commit 010dc172: - Update t_stream.c - 1 file changed, 2 insertions(+), 2 deletions(-) +antirez in commit ea1e1b12: + ACL LOG: test for AUTH reason. + 1 file changed, 9 insertions(+) -Mustafa Paltun in commit 6d0acb33: - Update help.h - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit 7379c78a: + ACL LOG: log failed auth attempts. + 5 files changed, 34 insertions(+), 12 deletions(-) -Oran Agra in commit de495ee7: - minor fix in creating a stream NACK for rdb and defrag tests - 2 files changed, 2 insertions(+), 2 deletions(-) +antirez in commit 9f6e84f6: + ACL LOG: implement a few basic tests. + 1 file changed, 87 insertions(+) -antirez in commit 71295ee3: - CLIENT UNBLOCK implemented. - 1 file changed, 22 insertions(+) +antirez in commit 82790e51: + ACL LOG: also log ACL errors in the scripting/MULTI ctx. + 2 files changed, 6 insertions(+), 2 deletions(-) -antirez in commit fb39bfd7: - Take clients in a ID -> Client handle dictionary. - 3 files changed, 6 insertions(+) +antirez in commit 943008eb: + ACL LOG: implement LOG RESET. + 1 file changed, 6 insertions(+), 2 deletions(-) -antirez in commit ed65d734: - CLIENT ID implemented. - 1 file changed, 4 insertions(+) +antirez in commit e271a611: + ACL LOG: group similar entries in a given time delta. + 1 file changed, 58 insertions(+), 3 deletions(-) -Salvatore Sanfilippo in commit 345b4809: - Merge pull request #5063 from oranagra/test_suite_improvements -Salvatore Sanfilippo in commit 35c5f3fa: - Merge pull request #5065 from oranagra/defrag_jemalloc5 -Oran Agra in commit 5616d4c6: - add active defrag support for streams - 6 files changed, 230 insertions(+), 25 deletions(-) - -Oran Agra in commit e8099cab: - add defrag hint support into jemalloc 5 - 3 files changed, 43 insertions(+) - -Oran Agra in commit 751eea24: - test suite infra improvements and fix - 2 files changed, 19 insertions(+) - -Salvatore Sanfilippo in commit bb666d44: - Merge pull request #5027 from tigertv/unstable -antirez in commit b9058c73: - Merge branch 'unstable' of github.com:/antirez/redis into unstable -antirez in commit 43831779: - Sentinel: test command renaming feature. - 1 file changed, 10 insertions(+) - -Salvatore Sanfilippo in commit eb052ba9: - Merge pull request #5059 from guybe7/fix_restore_warning -antirez in commit 27178a3f: - Fix type of argslen in sendSynchronousCommand(). - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit f1974d5d: + ACL LOG: actually emit entries. + 3 files changed, 34 insertions(+), 5 deletions(-) -antirez in commit 1f1e724f: - Remove black space. - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit d9b153c9: + ACL LOG: implement ACL LOG subcommadn skeleton. + 1 file changed, 37 insertions(+) -Salvatore Sanfilippo in commit aa5eaad4: - Merge pull request #5037 from madolson/repl-auth-fix -antirez in commit 3cf8dd2c: - Sentinel: fix SENTINEL SET error reporting. - 1 file changed, 18 insertions(+), 9 deletions(-) +antirez in commit 577fc438: + ACL LOG: data structures and initial functions. + 5 files changed, 54 insertions(+), 5 deletions(-) -Madelyn Olson in commit 45731edc: - Addressed comments +Leo Murillo in commit f7a94526: + Set ZSKIPLIST_MAXLEVEL to optimal value given 2^64 elements and p=0.25 1 file changed, 1 insertion(+), 1 deletion(-) -Madelyn Olson in commit e8d68b6b: - Fixed replication authentication with whitespace in password - 1 file changed, 12 insertions(+), 5 deletions(-) - -antirez in commit fc0c9c80: - Sentinel: drop the renamed-command entry in a more natural way. - 1 file changed, 4 insertions(+), 7 deletions(-) - -antirez in commit 8ba670f5: - Sentinel command renaming: document it into sentinel.conf. - 1 file changed, 19 insertions(+) - -antirez in commit a8a76bda: - Merge branch 'unstable' of github.com:/antirez/redis into unstable -antirez in commit 2358de68: - Sentinel command renaming: use case sensitive hashing for the dict. - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit a9c50088: - Sentinel command renaming: fix CONFIG SET event logging. - 1 file changed, 17 insertions(+), 1 deletion(-) - -antirez in commit b72cecd7: - Sentinel command renaming: fix CONFIG SET after refactoring. - 1 file changed, 5 insertions(+), 5 deletions(-) +WuYunlong in commit eecfa979: + Fix lua related memory leak. + 1 file changed, 1 insertion(+) -antirez in commit 91a384a5: - Sentinel command renaming: implement SENTINEL SET. - 1 file changed, 19 insertions(+) +WuYunlong in commit d2509811: + Add tcl regression test in scripting.tcl to reproduce memory leak. + 1 file changed, 5 insertions(+) -antirez in commit 903582dd: - Sentinel: make SENTINEL SET able to handle different arities. - 1 file changed, 19 insertions(+), 12 deletions(-) +Yossi Gottlieb in commit 29d4a150: + TLS: Fix missing initialization in redis-cli. + 1 file changed, 9 insertions(+) -antirez in commit c303e768: - Sentinel command renaming: config rewriting. - 1 file changed, 12 insertions(+) +Oran Agra in commit ec0c61da: + fix uninitialized info_cb var in module.c + 1 file changed, 1 insertion(+) -antirez in commit 60df7dbe: - Sentinel command renaming: rename-command option parsing. - 1 file changed, 11 insertions(+) +Guy Benoish in commit 6fe55c2f: + ld2string should fail if string contains \0 in the middle + 5 files changed, 20 insertions(+), 11 deletions(-) -antirez in commit 72e8a33b: - Sentinel command renaming: base machanism implemented. - 1 file changed, 64 insertions(+), 15 deletions(-) +antirez in commit bbce3ba9: + Add more info in the unblockClientFromModule() function. + 1 file changed, 7 insertions(+), 1 deletion(-) -Guy Benoish in commit dfcc20f4: - Fix compiler warning in restoreCommand - 1 file changed, 1 insertion(+), 1 deletion(-) +Guy Benoish in commit 40295fb3: + Modules: Fix blocked-client-related memory leak + 3 files changed, 51 insertions(+), 6 deletions(-) -Salvatore Sanfilippo in commit cf7fcdbe: - Merge pull request #4634 from soloestoy/special-auth -Salvatore Sanfilippo in commit 70b7fa2c: - Merge pull request #5049 from youjiali1995/fix-load-rdb -Salvatore Sanfilippo in commit 54d66d39: - Merge pull request #5053 from michael-grunder/zpopminmax-keypos -Salvatore Sanfilippo in commit 199e704a: - Merge pull request #5050 from shenlongxing/fix-typo -michael-grunder in commit db6b99f9: - Update ZPOPMIN/ZPOPMAX command declaration +antirez in commit 8e9d19bc: + Change error message for #6775. 1 file changed, 2 insertions(+), 2 deletions(-) -Salvatore Sanfilippo in commit a16aa03a: - Merge pull request #5051 from oranagra/streams_mem_estimate -Oran Agra in commit 20e10dc7: - fix streams memory estimation, missing raxSeek - 1 file changed, 2 insertions(+), 1 deletion(-) - -shenlongxing in commit ec55df11: - fix typo - 4 files changed, 4 insertions(+), 4 deletions(-) - -youjiali1995 in commit df6644fe: - Fix rdbLoadIntegerObject() to create shared objects when needed. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Salvatore Sanfilippo in commit 1527bcad: - Merge pull request #5036 from bepahol/unstable -Salvatore Sanfilippo in commit c1e82405: - Merge pull request #5039 from oranagra/rdb_dbsize_hint -Salvatore Sanfilippo in commit 79f55eed: - Merge pull request #5040 from oranagra/memrev64ifbe_fix -Salvatore Sanfilippo in commit c6f4118c: - Merge pull request #5045 from guybe7/restore_fix -Guy Benoish in commit b5197f1f: - Enhance RESTORE with RDBv9 new features - 5 files changed, 100 insertions(+), 22 deletions(-) - -Salvatore Sanfilippo in commit c6fdebf5: - Merge pull request #5042 from oranagra/malloc_usable_size_libc -Oran Agra in commit 482785ac: - add malloc_usable_size for libc malloc - 2 files changed, 8 insertions(+), 3 deletions(-) - -Salvatore Sanfilippo in commit 4da29630: - Merge pull request #5023 from FX-HAO/unstable -antirez in commit e7219025: - Test RDB stream encoding saving/loading. - 1 file changed, 17 insertions(+) - -Salvatore Sanfilippo in commit 5f5e1199: - Merge pull request #5041 from oranagra/redis-rdb-check_rdbLoadMillisecondTime -antirez in commit 4848fbec: - Modules: convert hash to hash table for big objects. - 1 file changed, 3 insertions(+) - -Oran Agra in commit f31b0405: - fix redis-rdb-check to provide proper arguments to rdbLoadMillisecondTime - 2 files changed, 3 insertions(+), 2 deletions(-) - -antirez in commit 333c98c4: - AOF: remove no longer used variable "now". - 1 file changed, 1 deletion(-) - -antirez in commit e94b2053: - Modify clusterRedirectClient() to handle ZPOP and XREAD. - 1 file changed, 5 insertions(+), 1 deletion(-) - -Oran Agra in commit 26229aa6: - use safe macro (non empty) in memrev64ifbe to eliminate empty if warning - 1 file changed, 3 insertions(+), 3 deletions(-) +Vasyl Melnychuk in commit ba146d4c: + Make error when submitting command in incorrect context more explicit + 1 file changed, 4 insertions(+), 1 deletion(-) -Oran Agra in commit 5cd3c952: - 64 bit RDB_OPCODE_RESIZEDB in rdb saving - 1 file changed, 3 insertions(+), 7 deletions(-) +antirez in commit 721a39dd: + Document I/O threads in redis.conf. + 1 file changed, 46 insertions(+) -antirez in commit ba92b517: - Remove AOF optimization to skip expired keys. - 1 file changed, 3 deletions(-) +antirez in commit 5be3a15a: + Setting N I/O threads should mean N-1 additional + 1 main thread. + 1 file changed, 25 insertions(+), 22 deletions(-) -Benjamin Holst in commit 36524060: - Update README.md - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit cbabf779: + Simplify #6379 changes. + 2 files changed, 4 insertions(+), 9 deletions(-) -antirez in commit 44571088: - Merge branch 'unstable' of github.com:/antirez/redis into unstable -antirez in commit 6967d0bd: - Revert fix #4976 just leaving the flush() part. - 1 file changed, 1 insertion(+), 5 deletions(-) +WuYunlong in commit 658749cc: + Free allocated sds in pfdebugCommand() to avoid memory leak. + 1 file changed, 1 insertion(+) -antirez in commit 0ed0dc3c: - Fix incrDecrCommand() to create shared objects when needed. - 1 file changed, 1 insertion(+), 1 deletion(-) +WuYunlong in commit 47988c96: + Fix potential memory leak of clusterLoadConfig(). + 1 file changed, 20 insertions(+), 5 deletions(-) -antirez in commit bd92389c: - Refactor createObjectFromLongLong() to be suitable for value objects. - 2 files changed, 33 insertions(+), 2 deletions(-) +WuYunlong in commit cc90f79b: + Fix potential memory leak of rioWriteBulkStreamID(). + 1 file changed, 4 insertions(+), 1 deletion(-) -Salvatore Sanfilippo in commit 3518bb66: - Merge pull request #5020 from shenlongxing/fix-config -antirez in commit 20766608: - Streams: fix xreadGetKeys() for correctness. - 1 file changed, 19 insertions(+), 5 deletions(-) +antirez in commit ecd17e81: + Jump to right label on AOF parsing error. + 1 file changed, 6 insertions(+), 4 deletions(-) -Salvatore Sanfilippo in commit e670ccff: - Merge pull request #4857 from youjiali1995/fix-command-getkeys -antirez in commit a0b27dae: - Streams: fix xreadGetKeys() buffer overflow. - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit 1927932b: + Port PR #6110 to new connection object code. + 1 file changed, 2 insertions(+), 2 deletions(-) -antirez in commit 62f9ac6f: - Streams: Change XADD MAXLEN handling of values <= 0. - 1 file changed, 3 insertions(+), 3 deletions(-) +antirez in commit f2df5773: + A few comments about main thread serving I/O as well. + 1 file changed, 7 insertions(+), 1 deletion(-) -Max Vetrov in commit d4c4f20a: - Update sort.c - 1 file changed, 1 insertion(+), 3 deletions(-) +zhaozhao.zz in commit b3ff8a4b: + Threaded IO: use main thread to handle read work + 1 file changed, 8 insertions(+), 1 deletion(-) -antirez in commit 79a1c19a: - XADD MAXLEN should return an error for values < 0. - 1 file changed, 5 insertions(+) +zhaozhao.zz in commit b1f2c510: + Threaded IO: use main thread to handle write work + 1 file changed, 10 insertions(+), 2 deletions(-) -Salvatore Sanfilippo in commit 2e0ab4a8: - Merge pull request #4976 from trevor211/fixDebugLoadaof -Salvatore Sanfilippo in commit 94658303: - Merge pull request #4758 from soloestoy/rdb-save-incremental-fsync -antirez in commit 6a66b93b: - Sentinel: add an option to deny online script reconfiguration. - 2 files changed, 41 insertions(+) - -antirez in commit d353023a: - Merge branch 'unstable' of github.com:/antirez/redis into unstable -antirez in commit d6e8fe77: - Fix infinite loop in dbRandomKey(). - 1 file changed, 13 insertions(+) +ShooterIT in commit 7bbafc56: + Rename rdb asynchronously + 1 file changed, 7 insertions(+) -Salvatore Sanfilippo in commit 40d5df65: - Merge pull request #5008 from zwkno1/unstable -Salvatore Sanfilippo in commit 8bc3ffcb: - Merge pull request #5021 from soloestoy/fix-exists -Salvatore Sanfilippo in commit 6c7847a1: - Merge pull request #5018 from soloestoy/optimize-reply -antirez in commit 1e92fde3: - Fix SCAN bug regression test, avoiding empty SREM call. +Leo Murillo in commit c7f75266: + Fix bug on KEYS command where pattern starts with * followed by \x00 (null char). 1 file changed, 1 insertion(+), 1 deletion(-) -Fuxin Hao in commit a4f658b2: - Fix update_zmalloc_stat_alloc in zrealloc +Jamie Scott in commit ed7ea13a: + Update to directive in redis.conf (missing s) 1 file changed, 1 insertion(+), 1 deletion(-) -================================================================================ -Redis 5.0 RC3 Released Wed Jun 14 9:51:44 CEST 2018 -================================================================================ - -Upgrade urgency LOW: - -This release fixes compilation of Redis RC2. For an error the commit from unstable -updating the Rax library was not merged into the 5.0 branch. - -================================================================================ -Redis 5.0 RC2 Released Wed Jun 13 12:49:13 CEST 2018 -================================================================================ - -Upgrade urgency CRITICAL: This release fixes important security issues. - HIGH: This release fixes a SCAN commands family bug. - MODERATE: This release fixes a PSYNC2 edge case with expires. - MODERATE: Sentinel related fixes. - LOW: All the other issues - -NOTE: This release breaks API compatibility with certain APIs that were -introduced in Redis 5.0 RC1. Notably ZPOP* now returns score/element in reverse -order. XINFO special form was removed, now XINFO STREAM must be -used to obtain general information about the stream. - -Redis 5.0 RC2 ixes a number of important issues: - -* Important security issues related to the Lua scripting engine. - Please check https://github.com/antirez/redis/issues/5017 - for more information. - -* A bug with SCAN, SSCAN, HSCAN and ZSCAN, that may not return all the elements. - We also add a regression test that can trigger the issue often when present, and - may in theory be able to find unrelated regressions. - -* A PSYNC2 bug is fixed: Redis should not expire keys when saving RDB files - because otherwise it is no longer possible to use such RDB file as a base - for partial resynchronization. It no longer represents the right state. - -* Compatibility of AOF with RDB preamble when the RDB checksum is disabled. - -* Sentinel bug that in some cases prevented Sentinel to detect that the master - was down immediately. A delay was added to the detection. - -* XREADGROUP would crash when the master had attacked slaves. - -* Replication and events generation of several streams command were fixed. - -* XREVRANGE fixed, in some cases it could not return elements, or crash the - server, or in general not behave correctly. - -* ZPOP can now unblock multiple clients in a sane way. - -* Other minor issues. - -Moreover this release adds new features: - -* XGROUP DESTROY and XGROUP SETID. - -* RDB loading speedup. - -* Configurable stream macro node limits (number of elements / bytes). - -* More smaller improvements. - -The following is the list of commits composing the release, please check -the commit messages and authors for credits. - -antirez in commit 9fdcc159: - Security: fix redis-cli buffer overflow. - 1 file changed, 16 insertions(+), 11 deletions(-) - -antirez in commit cf760071: - Security: fix Lua struct package offset handling. - 1 file changed, 6 insertions(+), 2 deletions(-) - -antirez in commit a57595ca: - Security: more cmsgpack fixes by @soloestoy. - 1 file changed, 7 insertions(+) - -antirez in commit 8783fb94: - Security: update Lua struct package for security. - 1 file changed, 23 insertions(+), 23 deletions(-) - -antirez in commit 8cb9344b: - Security: fix Lua cmsgpack library stack overflow. - 1 file changed, 3 insertions(+) - -赵磊 in commit 59080f60: - Fix dictScan(): It can't scan all buckets when dict is shrinking. - 1 file changed, 14 insertions(+), 11 deletions(-) +antirez in commit 3be77623: + Free fakeclient argv on AOF error. + 1 file changed, 11 insertions(+), 3 deletions(-) -dejun.xdj in commit ac2a824a: - Fix redis-cli memory leak when sending set preference command. +antirez in commit 15f6b748: + Git ignore: ignore more files. 1 file changed, 2 insertions(+) -dejun.xdj in commit c7197ff5: - Check if the repeat value is positive in while loop of cliSendCommand(). - 1 file changed, 1 insertion(+), 1 deletion(-) +Guy Benoish in commit 1b5bf40c: + Blocking XREAD[GROUP] should always reply with valid data (or timeout) + 3 files changed, 44 insertions(+), 10 deletions(-) -dejun.xdj in commit 3f77777f: - Change the type of repeat argument to long for function cliSendCommand. - 1 file changed, 1 insertion(+), 1 deletion(-) +John Sully in commit 954c20ed: + Add support for incremental build with header files + 2 files changed, 6 insertions(+), 1 deletion(-) -dejun.xdj in commit 7a565d72: - Fix negtive repeat command value issue. - 1 file changed, 11 insertions(+), 3 deletions(-) +WuYunlong in commit 11c3afd7: + Fix petential cluster link error. + 1 file changed, 4 insertions(+) + +Yossi Gottlieb in commit b752e83d: + Add REDISMODULE_CTX_FLAGS_MULTI_DIRTY. + 2 files changed, 8 insertions(+) -dejun.xdj in commit 64bf60fb: - Detect and stop saving history for auth command with repeat option. - 1 file changed, 17 insertions(+), 10 deletions(-) +hwware in commit e16eb874: + typo fix in acl.c + 1 file changed, 2 insertions(+), 2 deletions(-) -dejun.xdj in commit 5bed12aa: - Change the warning message a little bit to avoid trademark issuses. +Itamar Haber in commit 35ea9d23: + Adjusts 'io_threads_num' max to 128 1 file changed, 1 insertion(+), 1 deletion(-) -dejun.xdj in commit d71c4961: - Stop saving auth command in redis-cli history. +antirez in commit 38729126: + XCLAIM: Create the consumer only on successful claims. 1 file changed, 4 insertions(+), 2 deletions(-) -dejun.xdj in commit fca99e41: - Add warning message when using password on command line - 1 file changed, 1 insertion(+) - -antirez in commit 01407a3a: - Don't expire keys while loading RDB from AOF preamble. - 3 files changed, 5 insertions(+), 5 deletions(-) - -WuYunlong in commit fb5408cf: - Fix rdb save by allowing dumping of expire keys, so that when we add a new slave, and do a failover, eighter by manual or not, other local slaves will delete the expired keys properly. - 2 files changed, 3 insertions(+), 7 deletions(-) - -antirez in commit 0b8b6df4: - Backport hiredis issue 525 fix to compile on FreeBSD. +yz1509 in commit b9a15303: + avoid sentinel changes promoted_slave to be its own replica. 1 file changed, 1 insertion(+), 1 deletion(-) -antirez in commit e98627c5: - Add INIT INFO to the provided init script. - 1 file changed, 8 insertions(+) +antirez in commit 5e7e5e6b: + Fix active expire division by zero. + 1 file changed, 7 insertions(+), 4 deletions(-) + +antirez in commit e61dde88: + Fix duplicated CLIENT SETNAME reply. + 1 file changed, 1 deletion(-) -antirez in commit 17f5de89: - Fix ae.c when a timer finalizerProc adds an event. - 2 files changed, 10 insertions(+), 6 deletions(-) +Guy Benoish in commit cddf1da2: + Stream: Handle streamID-related edge cases + 4 files changed, 54 insertions(+), 4 deletions(-) -antirez in commit 266e6423: - Sentinel: fix delay in detecting ODOWN. - 1 file changed, 9 insertions(+), 5 deletions(-) +Oran Agra in commit 52ea44e5: + config.c adjust config limits and mutable + 2 files changed, 7 insertions(+), 7 deletions(-) -zhaozhao.zz in commit eafaf172: - AOF & RDB: be compatible with rdbchecksum no - 1 file changed, 9 insertions(+), 7 deletions(-) +antirez in commit 0f28ea16: + Inline protocol: handle empty strings well. + 1 file changed, 2 insertions(+), 6 deletions(-) -huijing.whj in commit 4630da37: - fix int overflow problem in freeMemoryIfNeeded - 1 file changed, 1 insertion(+), 1 deletion(-) +antirez in commit 00e5fefe: + Fix ip and missing mode in RM_GetClusterNodeInfo(). + 1 file changed, 5 insertions(+), 2 deletions(-) ================================================================================ -Redis 5.0 RC1 Released Tue May 29 14:14:11 CEST 2018 +Redis 6.0 RC1 Released Thu Dec 19 09:58:24 CEST 2019 ================================================================================ -Upgrade urgency LOW: This is the first RC of Redis 5. +Upgrade urgency LOW: This is the first RC of Redis 6. -Introduction to the Redis 5 release +Introduction to the Redis 6 release =================================== -Redis 5 is a release focused on a few important features. While Redis 4 -was very very focused on operations, Redis 5 changes are mostly user-facing, -with the implementation of new data types and operations on top of existing -types. The following are the major features of this release: - -1. The new Stream data type. https://redis.io/topics/streams-intro -2. New Redis modules APIs: Timers, Cluster and Dictionary APIs. -3. RDB now store LFU and LRU information. -4. The cluster manager was ported from Ruby (redis-trib.rb) to C code - inside redis-cli. Check `redis-cli --cluster help` for more info. -5. New sorted set commands: ZPOPMIN/MAX and blocking variants. -6. Active defragmentation version 2. -7. Improvemenets in HyperLogLog implementations. -8. Better memory reporting capabilities. -9. Many commands with sub-commands now have an HELP subcommand. -10. Better performances when clients connect and disconnect often. -11. Many bug fixes and other random improvements. -12. Jemalloc was upgraded to version 5.1 -13. CLIENT UNBLOCK and CLIENT ID. -14. The LOLWUT command was added. http://antirez.com/news/123 -15. We no longer use the "slave" word if not for API backward compatibility. -16. Differnet optimizations in the networking layer. -17. Lua improvements: - - Better propagation of Lua scripts to slaves / AOF. - - Lua scripts can now timeout and get in -BUSY state in the slave as well. -18. Dynamic HZ to balance idle CPU usage with responsiveness. -19. The Redis core was refactored and improved in many ways. +Redis 6 improves Redis in a number of key areas and is one of the largest +Redis releases in the history of the project, so here we'll list only +the biggest features in this release: + +* The modules system now has a number of new APIs that allow module authors + to make things otherwise not possible in the past. It is possible to + store arbitrary module private data in RDB files, to hook on different + server events, capture and rewrite commands executions, block clients on + keys, and so forth. +* The Redis active expire cycle was rewritten for much faster eviction of keys + that are already expired. Now the effort is tunable. +* Redis now supports SSL on all channels. +* ACL support, you can define users that can run only certain commands and/or + can only access only certain keys patterns. +* Redis now supports a new protocol called RESP3, which returns more + semantical replies: new clients using this protocol can understand just + from the reply what type to return to the calling program. +* There is server-side support for client-side caching of key values. This + feature is still experimental and will get more changes during the next + release candidates, but you can already test it and read about it here: + https://redis.io/topics/client-side-caching +* Redis can now optionally use threads to handle I/O, allowing to serve + 2 times as much operations per second in a single instance when + pipelining cannot be used. +* Diskless replication is now supported even on replicas: a replica is now + able, under certain conditions the user can configure, to load the RDB + in the first synchronization directly from the socket to the memory. +* Redis-benchmark now supports a Redis Cluster mode. +* SRANDMEMBER and similar commands have a better distribution. +* Redis-cli improvements. +* Systemd support rewritten. +* A Redis Cluster proxy was released here: + https://github.com/artix75/redis-cluster-proxy +* A Disque module for Redis was released here: + https://github.com/antirez/disque-module Thanks to all the users and developers who made this release possible. We'll follow up with more RC releases, until the code looks production ready and we don't get reports of serious issues for a while. A special thank you for the amount of work put into this release -(in decreasing number of commits) by: - -Fabio Nicotra, -Soloestoy -Itamar Haber -Oran Agra -Dvir Volk -dejun.xdj -Guy Benoish -Charsyam -Otmar Ertl -Jan-Erik Rediger -Spinlock - -Migrating from 4.0 to 5.0 +(in decreasing number of commits, only listing contributors with more +than a single commit) by: + + 685 antirez + 81 zhaozhao.zz + 76 Oran Agra + 51 artix + 28 Madelyn Olson + 27 Yossi Gottlieb + 15 David Carlier + 14 Guy Benoish + 14 Guy Korland + 13 Itamar Haber + 9 Angus Pearson + 8 WuYunlong + 8 yongman + 7 vattezhang + 7 Chris Lamb + 5 Dvir Volk + 5 meir@redislabs.com + 5 chendianqiang + 5 John Sully + 4 dejun.xdj + 4 Daniel Dai + 4 Johannes Truschnigg + 4 swilly22 + 3 Bruce Merry + 3 filipecosta90 + 3 youjiali1995 + 2 James Rouzier + 2 Andrey Bugaevskiy + 2 Brad Solomon + 2 Hamid Alaei + 2 Michael Chaten + 2 Steve Webster + 2 Wander Hillen + 2 Weiliang Li + 2 Yuan Zhou + 2 charsyam + 2 hujie + 2 jem + 2 shenlongxing + 2 valentino + 2 zhudacai 00228490 + 2 喜欢兰花山丘 + +Migrating from 5.0 to 6.0 ========================= -Redis 4.0 is mostly a strict subset of 5.0, you should not have any problem -upgrading your application from 4.0 to 5.0. However this is a list of small -non-backward compatible changes introduced in the 5.0 release: - -* redis-cli now implements the cluster management tool. We still ship the - old redis-trib, but new fixes will be implemented only in redis-cli. - See `redis-cli --cluster help` for more info. - -* The RDB format changed. Redis 5.0 is still able to read 4.0 (and all the - past versions) files, but not the other way around. - -* Certain log formats and sentences are different in Redis 5.0. - -* Now by default maxmemory is ignored by slaves, and used only once a slave - is promoted to master. It means that in setups where you want slaves to - enforce maxmemory in an independent way from the master (that will anyway - stream the key eviction DEL commands), you should active this feature manually - and make sure you understand that it breaks consistency if writes are not - always idempotent. TLDR: the new behavior is much better for 99.999% of use - cases, revert it if you really know what you are doing. - -* Scripts are only replicated by their *effects* and not by sending EVAL/EVALSHA - to slaves or the AOF log itself. This is much better in the general case - and in the future we want to totally remove the other possiblity of - propagating scripts the old way (as EVAL). However you can still turn this - back to the default via the non-documented (if not here) Redis configuration - directive "lua-replicate-commands yes" or - "DEBUG lua-always-replicate-commands 0". However note that Redis 6 may - completely remove such feature. - -* Because of the above change related to scripts replication, certain Redis - commands that in Redis 4 had their result ordered lexicographically before - being passed to Lua via the return value of redis.call(), now have a behavior - more similar to calling the commands directly from a normal client. For - instance the ordering of elements returned by SMEMBERS or SDIFF is now - undetermined in Lua, exactly as it is by default when calling the commands - from a non-scripting context. +Redis 6.0 is mostly a strict superset of 5.0, you should not have any problem +upgrading your application from 5.0 to 6.0. However this is a list of small +non-backward compatible changes introduced in the 6.0 release: + +* The SPOP command no longer returns null when the set key does not + exist. Now it returns the empty set as it should and as happens when it is + called with a 0 argument. This is technically a fix, however it changes the + old behavior. -------------------------------------------------------------------------------- diff --git a/redis.submodule/README.md b/redis.submodule/README.md index 4b1a983..c080134 100644 --- a/redis.submodule/README.md +++ b/redis.submodule/README.md @@ -35,6 +35,11 @@ It is as simple as: % make +To build with TLS support, you'll need OpenSSL development libraries (e.g. +libssl-dev on Debian/Ubuntu) and run: + + % make BUILD_TLS=yes + You can run a 32 bit Redis binary using: % make 32bit @@ -43,6 +48,13 @@ After building Redis, it is a good idea to test it using: % make test +If TLS is built, running the tests with TLS enabled (you will need `tcl-tls` +installed): + + % ./utils/gen-test-certs.sh + % ./runtest --tls + + Fixing build problems with dependencies or cached build options --------- @@ -125,6 +137,12 @@ as options using the command line. Examples: All the options in redis.conf are also supported as options using the command line, with exactly the same name. +Running Redis with TLS: +------------------ + +Please consult the [TLS.md](TLS.md) file for more information on +how to use Redis with TLS. + Playing with Redis ------------------ @@ -166,6 +184,8 @@ for Ubuntu and Debian systems: % cd utils % ./install_server.sh +_Note_: `install_server.sh` will not work on Mac OSX; it is built for Linux only. + The script will ask you a few questions and will setup everything you need to run Redis properly as a background daemon that will start again on system reboots. @@ -216,7 +236,7 @@ Inside the root are the following important directories: * `src`: contains the Redis implementation, written in C. * `tests`: contains the unit tests, implemented in Tcl. -* `deps`: contains libraries Redis uses. Everything needed to compile Redis is inside this directory; your system just needs to provide `libc`, a POSIX compatible interface and a C compiler. Notably `deps` contains a copy of `jemalloc`, which is the default allocator of Redis under Linux. Note that under `deps` there are also things which started with the Redis project, but for which the main repository is not `antirez/redis`. An exception to this rule is `deps/geohash-int` which is the low level geocoding library used by Redis: it originated from a different project, but at this point it diverged so much that it is developed as a separated entity directly inside the Redis repository. +* `deps`: contains libraries Redis uses. Everything needed to compile Redis is inside this directory; your system just needs to provide `libc`, a POSIX compatible interface and a C compiler. Notably `deps` contains a copy of `jemalloc`, which is the default allocator of Redis under Linux. Note that under `deps` there are also things which started with the Redis project, but for which the main repository is not `antirez/redis`. There are a few more directories but they are not very important for our goals here. We'll focus mostly on `src`, where the Redis implementation is contained, @@ -404,7 +424,7 @@ replicas, or to continue the replication after a disconnection. Other C files --- -* `t_hash.c`, `t_list.c`, `t_set.c`, `t_string.c` and `t_zset.c` contains the implementation of the Redis data types. They implement both an API to access a given data type, and the client commands implementations for these data types. +* `t_hash.c`, `t_list.c`, `t_set.c`, `t_string.c`, `t_zset.c` and `t_stream.c` contains the implementation of the Redis data types. They implement both an API to access a given data type, and the client commands implementations for these data types. * `ae.c` implements the Redis event loop, it's a self contained library which is simple to read and understand. * `sds.c` is the Redis string library, check http://github.com/antirez/sds for more information. * `anet.c` is a library to use POSIX networking in a simpler way compared to the raw interface exposed by the kernel. diff --git a/redis.submodule/TLS.md b/redis.submodule/TLS.md new file mode 100644 index 0000000..e480c1e --- /dev/null +++ b/redis.submodule/TLS.md @@ -0,0 +1,89 @@ +TLS Support +=========== + +Getting Started +--------------- + +### Building + +To build with TLS support you'll need OpenSSL development libraries (e.g. +libssl-dev on Debian/Ubuntu). + +Run `make BUILD_TLS=yes`. + +### Tests + +To run Redis test suite with TLS, you'll need TLS support for TCL (i.e. +`tcl-tls` package on Debian/Ubuntu). + +1. Run `./utils/gen-test-certs.sh` to generate a root CA and a server + certificate. + +2. Run `./runtest --tls` or `./runtest-cluster --tls` to run Redis and Redis + Cluster tests in TLS mode. + +### Running manually + +To manually run a Redis server with TLS mode (assuming `gen-test-certs.sh` was +invoked so sample certificates/keys are available): + + ./src/redis-server --tls-port 6379 --port 0 \ + --tls-cert-file ./tests/tls/redis.crt \ + --tls-key-file ./tests/tls/redis.key \ + --tls-ca-cert-file ./tests/tls/ca.crt + +To connect to this Redis server with `redis-cli`: + + ./src/redis-cli --tls \ + --cert ./tests/tls/redis.crt \ + --key ./tests/tls/redis.key \ + --cacert ./tests/tls/ca.crt + +This will disable TCP and enable TLS on port 6379. It's also possible to have +both TCP and TLS available, but you'll need to assign different ports. + +To make a Replica connect to the master using TLS, use `--tls-replication yes`, +and to make Redis Cluster use TLS across nodes use `--tls-cluster yes`. + +Connections +----------- + +All socket operations now go through a connection abstraction layer that hides +I/O and read/write event handling from the caller. + +**Multi-threading I/O is not currently supported for TLS**, as a TLS connection +needs to do its own manipulation of AE events which is not thread safe. The +solution is probably to manage independent AE loops for I/O threads and longer +term association of connections with threads. This may potentially improve +overall performance as well. + +Sync IO for TLS is currently implemented in a hackish way, i.e. making the +socket blocking and configuring socket-level timeout. This means the timeout +value may not be so accurate, and there would be a lot of syscall overhead. +However I believe that getting rid of syncio completely in favor of pure async +work is probably a better move than trying to fix that. For replication it would +probably not be so hard. For cluster keys migration it might be more difficult, +but there are probably other good reasons to improve that part anyway. + +To-Do List +---------- + +- [ ] Add session caching support. Check if/how it's handled by clients to + assess how useful/important it is. +- [ ] redis-benchmark support. The current implementation is a mix of using + hiredis for parsing and basic networking (establishing connections), but + directly manipulating sockets for most actions. This will need to be cleaned + up for proper TLS support. The best approach is probably to migrate to hiredis + async mode. +- [ ] redis-cli `--slave` and `--rdb` support. + +Multi-port +---------- + +Consider the implications of allowing TLS to be configured on a separate port, +making Redis listening on multiple ports: + +1. Startup banner port notification +2. Proctitle +3. How slaves announce themselves +4. Cluster bus port calculation diff --git a/redis.submodule/deps/Makefile b/redis.submodule/deps/Makefile index eb35c1e..700867f 100644 --- a/redis.submodule/deps/Makefile +++ b/redis.submodule/deps/Makefile @@ -41,9 +41,13 @@ distclean: .PHONY: distclean +ifeq ($(BUILD_TLS),yes) + HIREDIS_MAKE_FLAGS = USE_SSL=1 +endif + hiredis: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) - cd hiredis && $(MAKE) static + cd hiredis && $(MAKE) static $(HIREDIS_MAKE_FLAGS) .PHONY: hiredis diff --git a/redis.submodule/deps/README.md b/redis.submodule/deps/README.md index 367ee16..685dbb4 100644 --- a/redis.submodule/deps/README.md +++ b/redis.submodule/deps/README.md @@ -2,7 +2,6 @@ This directory contains all Redis dependencies, except for the libc that should be provided by the operating system. * **Jemalloc** is our memory allocator, used as replacement for libc malloc on Linux by default. It has good performances and excellent fragmentation behavior. This component is upgraded from time to time. -* **geohash-int** is inside the dependencies directory but is actually part of the Redis project, since it is our private fork (heavily modified) of a library initially developed for Ardb, which is in turn a fork of Redis. * **hiredis** is the official C client library for Redis. It is used by redis-cli, redis-benchmark and Redis Sentinel. It is part of the Redis official ecosystem but is developed externally from the Redis repository, so we just upgrade it as needed. * **linenoise** is a readline replacement. It is developed by the same authors of Redis but is managed as a separated project and updated as needed. * **lua** is Lua 5.1 with minor changes for security and additional libraries. @@ -42,11 +41,6 @@ the following additional steps: changed, otherwise you could just copy the old implementation if you are upgrading just to a similar version of Jemalloc. -Geohash ---- - -This is never upgraded since it's part of the Redis project. If there are changes to merge from Ardb there is the need to manually check differences, but at this point the source code is pretty different. - Hiredis --- diff --git a/redis.submodule/deps/hiredis/.gitignore b/redis.submodule/deps/hiredis/.gitignore index c44b5c5..8e50b54 100644 --- a/redis.submodule/deps/hiredis/.gitignore +++ b/redis.submodule/deps/hiredis/.gitignore @@ -5,3 +5,4 @@ /*.dylib /*.a /*.pc +*.dSYM diff --git a/redis.submodule/deps/hiredis/.travis.yml b/redis.submodule/deps/hiredis/.travis.yml index ad08076..dd8e0e7 100644 --- a/redis.submodule/deps/hiredis/.travis.yml +++ b/redis.submodule/deps/hiredis/.travis.yml @@ -8,6 +8,12 @@ os: - linux - osx +branches: + only: + - staging + - trying + - master + before_script: - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then brew update; brew install redis; fi @@ -20,20 +26,72 @@ addons: - libc6-dev-i386 - libc6-dbg:i386 - gcc-multilib + - g++-multilib - valgrind env: - - CFLAGS="-Werror" - - PRE="valgrind --track-origins=yes --leak-check=full" - - TARGET="32bit" TARGET_VARS="32bit-vars" CFLAGS="-Werror" - - TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" + - BITS="32" + - BITS="64" -matrix: - exclude: - - os: osx - env: PRE="valgrind --track-origins=yes --leak-check=full" +script: + - EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DHIREDIS_SSL:BOOL=ON"; + if [ "$TRAVIS_OS_NAME" == "osx" ]; then + if [ "$BITS" == "32" ]; then + CFLAGS="-m32 -Werror"; + CXXFLAGS="-m32 -Werror"; + LDFLAGS="-m32"; + EXTRA_CMAKE_OPTS=; + else + CFLAGS="-Werror"; + CXXFLAGS="-Werror"; + fi; + else + TEST_PREFIX="valgrind --track-origins=yes --leak-check=full"; + if [ "$BITS" == "32" ]; then + CFLAGS="-m32 -Werror"; + CXXFLAGS="-m32 -Werror"; + LDFLAGS="-m32"; + EXTRA_CMAKE_OPTS=; + else + CFLAGS="-Werror"; + CXXFLAGS="-Werror"; + fi; + fi; + export CFLAGS CXXFLAGS LDFLAGS TEST_PREFIX EXTRA_CMAKE_OPTS + - mkdir build/ && cd build/ + - cmake .. ${EXTRA_CMAKE_OPTS} + - make VERBOSE=1 + - ctest -V - - os: osx - env: TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" +matrix: + include: + # Windows MinGW cross compile on Linux + - os: linux + dist: xenial + compiler: mingw + addons: + apt: + packages: + - ninja-build + - gcc-mingw-w64-x86-64 + - g++-mingw-w64-x86-64 + script: + - mkdir build && cd build + - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_BUILD_WITH_INSTALL_RPATH=on + - ninja -v -script: make $TARGET CFLAGS="$CFLAGS" && make check PRE="$PRE" && make $TARGET_VARS hiredis-example + # Windows MSVC 2017 + - os: windows + compiler: msvc + env: + - MATRIX_EVAL="CC=cl.exe && CXX=cl.exe" + before_install: + - eval "${MATRIX_EVAL}" + install: + - choco install ninja + script: + - mkdir build && cd build + - cmd.exe /C '"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" amd64 && + cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release && + ninja -v' + - ctest -V diff --git a/redis.submodule/deps/hiredis/CHANGELOG.md b/redis.submodule/deps/hiredis/CHANGELOG.md index f92bcb3..d1d37e5 100644 --- a/redis.submodule/deps/hiredis/CHANGELOG.md +++ b/redis.submodule/deps/hiredis/CHANGELOG.md @@ -1,12 +1,18 @@ ### 1.0.0 (unreleased) -**Fixes**: +**BREAKING CHANGES**: -* Catch a buffer overflow when formatting the error message -* Import latest upstream sds. This breaks applications that are linked against the old hiredis v0.13 -* Fix warnings, when compiled with -Wshadow -* Make hiredis compile in Cygwin on Windows, now CI-tested +* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now + protocol errors. This is consistent with the RESP specification. On 32-bit + platforms, the upper bound is lowered to `SIZE_MAX`. + +* Change `redisReply.len` to `size_t`, as it denotes the the size of a string + User code should compare this to `size_t` values as well. If it was used to + compare to other values, casting might be necessary or can be removed, if + casting was applied before. + +### 0.x.x (unreleased) **BREAKING CHANGES**: * Change `redisReply.len` to `size_t`, as it denotes the the size of a string @@ -14,6 +20,50 @@ User code should compare this to `size_t` values as well. If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before. +* `redisReplyObjectFunctions.createArray` now takes `size_t` for its length parameter. + +### 0.14.0 (2018-09-25) + +* Make string2ll static to fix conflict with Redis (Tom Lee [c3188b]) +* Use -dynamiclib instead of -shared for OSX (Ryan Schmidt [a65537]) +* Use string2ll from Redis w/added tests (Michael Grunder [7bef04, 60f622]) +* Makefile - OSX compilation fixes (Ryan Schmidt [881fcb, 0e9af8]) +* Remove redundant NULL checks (Justin Brewer [54acc8, 58e6b8]) +* Fix bulk and multi-bulk length truncation (Justin Brewer [109197]) +* Fix SIGSEGV in OpenBSD by checking for NULL before calling freeaddrinfo (Justin Brewer [546d94]) +* Several POSIX compatibility fixes (Justin Brewer [bbeab8, 49bbaa, d1c1b6]) +* Makefile - Compatibility fixes (Dimitri Vorobiev [3238cf, 12a9d1]) +* Makefile - Fix make install on FreeBSD (Zach Shipko [a2ef2b]) +* Makefile - don't assume $(INSTALL) is cp (Igor Gnatenko [725a96]) +* Separate side-effect causing function from assert and small cleanup (amallia [b46413, 3c3234]) +* Don't send negative values to `__redisAsyncCommand` (Frederik Deweerdt [706129]) +* Fix leak if setsockopt fails (Frederik Deweerdt [e21c9c]) +* Fix libevent leak (zfz [515228]) +* Clean up GCC warning (Ichito Nagata [2ec774]) +* Keep track of errno in `__redisSetErrorFromErrno()` as snprintf may use it (Jin Qing [25cd88]) +* Solaris compilation fix (Donald Whyte [41b07d]) +* Reorder linker arguments when building examples (Tustfarm-heart [06eedd]) +* Keep track of subscriptions in case of rapid subscribe/unsubscribe (Hyungjin Kim [073dc8, be76c5, d46999]) +* libuv use after free fix (Paul Scott [cbb956]) +* Properly close socket fd on reconnect attempt (WSL [64d1ec]) +* Skip valgrind in OSX tests (Jan-Erik Rediger [9deb78]) +* Various updates for Travis testing OSX (Ted Nyman [fa3774, 16a459, bc0ea5]) +* Update libevent (Chris Xin [386802]) +* Change sds.h for building in C++ projects (Ali Volkan ATLI [f5b32e]) +* Use proper format specifier in redisFormatSdsCommandArgv (Paulino Huerta, Jan-Erik Rediger [360a06, 8655a6]) +* Better handling of NULL reply in example code (Jan-Erik Rediger [1b8ed3]) +* Prevent overflow when formatting an error (Jan-Erik Rediger [0335cb]) +* Compatibility fix for strerror_r (Tom Lee [bb1747]) +* Properly detect integer parse/overflow errors (Justin Brewer [93421f]) +* Adds CI for Windows and cygwin fixes (owent, [6c53d6, 6c3e40]) +* Catch a buffer overflow when formatting the error message +* Import latest upstream sds. This breaks applications that are linked against the old hiredis v0.13 +* Fix warnings, when compiled with -Wshadow +* Make hiredis compile in Cygwin on Windows, now CI-tested +* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now + protocol errors. This is consistent with the RESP specification. On 32-bit + platforms, the upper bound is lowered to `SIZE_MAX`. + * Remove backwards compatibility macro's This removes the following old function aliases, use the new name now: @@ -94,7 +144,7 @@ The parser, standalone since v0.12.0, can now be compiled on Windows * Add IPv6 support -* Remove possiblity of multiple close on same fd +* Remove possibility of multiple close on same fd * Add ability to bind source address on connect diff --git a/redis.submodule/deps/hiredis/CMakeLists.txt b/redis.submodule/deps/hiredis/CMakeLists.txt new file mode 100644 index 0000000..9e78894 --- /dev/null +++ b/redis.submodule/deps/hiredis/CMakeLists.txt @@ -0,0 +1,90 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 3.4.0) +INCLUDE(GNUInstallDirs) +PROJECT(hiredis) + +OPTION(ENABLE_SSL "Build hiredis_ssl for SSL support" OFF) + +MACRO(getVersionBit name) + SET(VERSION_REGEX "^#define ${name} (.+)$") + FILE(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/hiredis.h" + VERSION_BIT REGEX ${VERSION_REGEX}) + STRING(REGEX REPLACE ${VERSION_REGEX} "\\1" ${name} "${VERSION_BIT}") +ENDMACRO(getVersionBit) + +getVersionBit(HIREDIS_MAJOR) +getVersionBit(HIREDIS_MINOR) +getVersionBit(HIREDIS_PATCH) +getVersionBit(HIREDIS_SONAME) +SET(VERSION "${HIREDIS_MAJOR}.${HIREDIS_MINOR}.${HIREDIS_PATCH}") +MESSAGE("Detected version: ${VERSION}") + +PROJECT(hiredis VERSION "${VERSION}") + +SET(ENABLE_EXAMPLES OFF CACHE BOOL "Enable building hiredis examples") + +ADD_LIBRARY(hiredis SHARED + async.c + dict.c + hiredis.c + net.c + read.c + sds.c + sockcompat.c) + +SET_TARGET_PROPERTIES(hiredis + PROPERTIES + VERSION "${HIREDIS_SONAME}") +IF(WIN32 OR MINGW) + TARGET_LINK_LIBRARIES(hiredis PRIVATE ws2_32) +ENDIF() +TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC .) + +CONFIGURE_FILE(hiredis.pc.in hiredis.pc @ONLY) + +INSTALL(TARGETS hiredis + DESTINATION "${CMAKE_INSTALL_LIBDIR}") + +INSTALL(FILES hiredis.h read.h sds.h async.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + +INSTALL(DIRECTORY adapters + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + +IF(ENABLE_SSL) + IF (NOT OPENSSL_ROOT_DIR) + IF (APPLE) + SET(OPENSSL_ROOT_DIR "/usr/local/opt/openssl") + ENDIF() + ENDIF() + FIND_PACKAGE(OpenSSL REQUIRED) + ADD_LIBRARY(hiredis_ssl SHARED + ssl.c) + TARGET_INCLUDE_DIRECTORIES(hiredis_ssl PRIVATE "${OPENSSL_INCLUDE_DIR}") + TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE ${OPENSSL_LIBRARIES}) + CONFIGURE_FILE(hiredis_ssl.pc.in hiredis_ssl.pc @ONLY) + + INSTALL(TARGETS hiredis_ssl + DESTINATION "${CMAKE_INSTALL_LIBDIR}") + + INSTALL(FILES hiredis_ssl.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) +ENDIF() + +IF(NOT (WIN32 OR MINGW)) + ENABLE_TESTING() + ADD_EXECUTABLE(hiredis-test test.c) + TARGET_LINK_LIBRARIES(hiredis-test hiredis) + ADD_TEST(NAME hiredis-test + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test.sh) +ENDIF() + +# Add examples +IF(ENABLE_EXAMPLES) + ADD_SUBDIRECTORY(examples) +ENDIF(ENABLE_EXAMPLES) diff --git a/redis.submodule/deps/hiredis/Makefile b/redis.submodule/deps/hiredis/Makefile index 9a4de83..25ac154 100644 --- a/redis.submodule/deps/hiredis/Makefile +++ b/redis.submodule/deps/hiredis/Makefile @@ -3,11 +3,17 @@ # Copyright (C) 2010-2011 Pieter Noordhuis # This file is released under the BSD license, see the COPYING file -OBJ=net.o hiredis.o sds.o async.o read.o +OBJ=net.o hiredis.o sds.o async.o read.o sockcompat.o +SSL_OBJ=ssl.o EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib +ifeq ($(USE_SSL),1) +EXAMPLES+=hiredis-example-ssl hiredis-example-libevent-ssl +endif TESTS=hiredis-test LIBNAME=libhiredis +SSL_LIBNAME=libhiredis_ssl PKGCONFNAME=hiredis.pc +SSL_PKGCONFNAME=hiredis_ssl.pc HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}') HIREDIS_MINOR=$(shell grep HIREDIS_MINOR hiredis.h | awk '{print $$3}') @@ -36,71 +42,109 @@ endef export REDIS_TEST_CONFIG # Fallback to gcc when $CC is not in $PATH. -CC:=$(shell sh -c 'type $(CC) >/dev/null 2>/dev/null && echo $(CC) || echo gcc') -CXX:=$(shell sh -c 'type $(CXX) >/dev/null 2>/dev/null && echo $(CXX) || echo g++') +CC:=$(shell sh -c 'type $${CC%% *} >/dev/null 2>/dev/null && echo $(CC) || echo gcc') +CXX:=$(shell sh -c 'type $${CXX%% *} >/dev/null 2>/dev/null && echo $(CXX) || echo g++') OPTIMIZATION?=-O3 -WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings +WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings -Wno-missing-field-initializers DEBUG_FLAGS?= -g -ggdb -REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CFLAGS) $(WARNINGS) $(DEBUG_FLAGS) $(ARCH) -REAL_LDFLAGS=$(LDFLAGS) $(ARCH) +REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CPPFLAGS) $(CFLAGS) $(WARNINGS) $(DEBUG_FLAGS) +REAL_LDFLAGS=$(LDFLAGS) DYLIBSUFFIX=so STLIBSUFFIX=a DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME) DYLIB_MAJOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR) DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX) -DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) +SSL_DYLIBNAME=$(SSL_LIBNAME).$(DYLIBSUFFIX) +DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) STLIBNAME=$(LIBNAME).$(STLIBSUFFIX) -STLIB_MAKE_CMD=ar rcs $(STLIBNAME) +SSL_STLIBNAME=$(SSL_LIBNAME).$(STLIBSUFFIX) +STLIB_MAKE_CMD=$(AR) rcs # Platform-specific overrides uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') + +USE_SSL?=0 + +# This is required for test.c only +ifeq ($(USE_SSL),1) + CFLAGS+=-DHIREDIS_TEST_SSL +endif + +ifeq ($(uname_S),Linux) + SSL_LDFLAGS=-lssl -lcrypto +else + OPENSSL_PREFIX?=/usr/local/opt/openssl + CFLAGS+=-I$(OPENSSL_PREFIX)/include + SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto +endif + ifeq ($(uname_S),SunOS) REAL_LDFLAGS+= -ldl -lnsl -lsocket DYLIB_MAKE_CMD=$(CC) -G -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS) - INSTALL= cp -r endif ifeq ($(uname_S),Darwin) DYLIBSUFFIX=dylib DYLIB_MINOR_NAME=$(LIBNAME).$(HIREDIS_SONAME).$(DYLIBSUFFIX) - DYLIB_MAKE_CMD=$(CC) -shared -Wl,-install_name,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) + DYLIB_MAKE_CMD=$(CC) -dynamiclib -Wl,-install_name,$(PREFIX)/$(LIBRARY_PATH)/$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) endif all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME) +ifeq ($(USE_SSL),1) +all: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME) +endif # Deps (use make dep to generate this) async.o: async.c fmacros.h async.h hiredis.h read.h sds.h net.h dict.c dict.h dict.o: dict.c fmacros.h dict.h -hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h -net.o: net.c fmacros.h net.h hiredis.h read.h sds.h +hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h win32.h +net.o: net.c fmacros.h net.h hiredis.h read.h sds.h sockcompat.h win32.h read.o: read.c fmacros.h read.h sds.h sds.o: sds.c sds.h +sockcompat.o: sockcompat.c sockcompat.h +ssl.o: ssl.c hiredis.h test.o: test.c fmacros.h hiredis.h read.h sds.h $(DYLIBNAME): $(OBJ) - $(DYLIB_MAKE_CMD) $(OBJ) + $(DYLIB_MAKE_CMD) -o $(DYLIBNAME) $(OBJ) $(REAL_LDFLAGS) $(STLIBNAME): $(OBJ) - $(STLIB_MAKE_CMD) $(OBJ) + $(STLIB_MAKE_CMD) $(STLIBNAME) $(OBJ) + +$(SSL_DYLIBNAME): $(SSL_OBJ) + $(DYLIB_MAKE_CMD) -o $(SSL_DYLIBNAME) $(SSL_OBJ) $(REAL_LDFLAGS) $(SSL_LDFLAGS) + +$(SSL_STLIBNAME): $(SSL_OBJ) + $(STLIB_MAKE_CMD) $(SSL_STLIBNAME) $(SSL_OBJ) dynamic: $(DYLIBNAME) static: $(STLIBNAME) +ifeq ($(USE_SSL),1) +dynamic: $(SSL_DYLIBNAME) +static: $(SSL_STLIBNAME) +endif # Binaries: hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -levent $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(REAL_LDFLAGS) + +hiredis-example-libevent-ssl: examples/example-libevent-ssl.c adapters/libevent.h $(STLIBNAME) $(SSL_STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) hiredis-example-libev: examples/example-libev.c adapters/libev.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -lev $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -lev $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-glib: examples/example-glib.c adapters/glib.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) $(shell pkg-config --cflags --libs glib-2.0) -I. $< $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(shell pkg-config --cflags --libs glib-2.0) $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-ivykis: examples/example-ivykis.c adapters/ivykis.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -livykis $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -livykis $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) $(REAL_LDFLAGS) + +hiredis-example-ssl: examples/example-ssl.c $(STLIBNAME) $(SSL_STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) ifndef AE_DIR hiredis-example-ae: @@ -117,7 +161,7 @@ hiredis-example-libuv: @false else hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) $(REAL_LDFLAGS) endif ifeq ($(and $(QT_MOC),$(QT_INCLUDE_DIR),$(QT_LIBRARY_DIR)),) @@ -134,38 +178,35 @@ hiredis-example-qt: examples/example-qt.cpp adapters/qt.h $(STLIBNAME) endif hiredis-example: examples/example.c $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS) examples: $(EXAMPLES) -hiredis-test: test.o $(STLIBNAME) +TEST_LIBS = $(STLIBNAME) +ifeq ($(USE_SSL),1) + TEST_LIBS += $(SSL_STLIBNAME) -lssl -lcrypto -lpthread +endif +hiredis-test: test.o $(TEST_LIBS) hiredis-%: %.o $(STLIBNAME) - $(CC) $(REAL_CFLAGS) -o $@ $(REAL_LDFLAGS) $< $(STLIBNAME) + $(CC) $(REAL_CFLAGS) -o $@ $< $(TEST_LIBS) $(REAL_LDFLAGS) test: hiredis-test ./hiredis-test check: hiredis-test - @echo "$$REDIS_TEST_CONFIG" | $(REDIS_SERVER) - - $(PRE) ./hiredis-test -h 127.0.0.1 -p $(REDIS_PORT) -s /tmp/hiredis-test-redis.sock || \ - ( kill `cat /tmp/hiredis-test-redis.pid` && false ) - kill `cat /tmp/hiredis-test-redis.pid` + TEST_SSL=$(USE_SSL) ./test.sh .c.o: $(CC) -std=c99 -pedantic -c $(REAL_CFLAGS) $< clean: - rm -rf $(DYLIBNAME) $(STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov + rm -rf $(DYLIBNAME) $(STLIBNAME) $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov dep: - $(CC) -MM *.c + $(CC) $(CPPFLAGS) $(CFLAGS) -MM *.c -ifeq ($(uname_S),SunOS) - INSTALL?= cp -r -endif - -INSTALL?= cp -a +INSTALL?= cp -pPR $(PKGCONFNAME): hiredis.h @echo "Generating $@ for pkgconfig..." @@ -180,9 +221,24 @@ $(PKGCONFNAME): hiredis.h @echo Libs: -L\$${libdir} -lhiredis >> $@ @echo Cflags: -I\$${includedir} -D_FILE_OFFSET_BITS=64 >> $@ +$(SSL_PKGCONFNAME): hiredis.h + @echo "Generating $@ for pkgconfig..." + @echo prefix=$(PREFIX) > $@ + @echo exec_prefix=\$${prefix} >> $@ + @echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@ + @echo includedir=$(PREFIX)/$(INCLUDE_PATH) >> $@ + @echo >> $@ + @echo Name: hiredis_ssl >> $@ + @echo Description: SSL Support for hiredis. >> $@ + @echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@ + @echo Requires: hiredis >> $@ + @echo Libs: -L\$${libdir} -lhiredis_ssl >> $@ + @echo Libs.private: -lssl -lcrypto >> $@ + install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) - mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_LIBRARY_PATH) - $(INSTALL) hiredis.h async.h read.h sds.h adapters $(INSTALL_INCLUDE_PATH) + mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL_LIBRARY_PATH) + $(INSTALL) hiredis.h async.h read.h sds.h $(INSTALL_INCLUDE_PATH) + $(INSTALL) adapters/*.h $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL) $(DYLIBNAME) $(INSTALL_LIBRARY_PATH)/$(DYLIB_MINOR_NAME) cd $(INSTALL_LIBRARY_PATH) && ln -sf $(DYLIB_MINOR_NAME) $(DYLIBNAME) $(INSTALL) $(STLIBNAME) $(INSTALL_LIBRARY_PATH) diff --git a/redis.submodule/deps/hiredis/README.md b/redis.submodule/deps/hiredis/README.md index 01223ea..c0b432f 100644 --- a/redis.submodule/deps/hiredis/README.md +++ b/redis.submodule/deps/hiredis/README.md @@ -286,6 +286,7 @@ return `REDIS_ERR`. The function to set the disconnect callback has the followin ```c int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); ``` +`ac->data` may be used to pass user data to this callback, the same can be done for redisConnectCallback. ### Sending commands and their callbacks In an asynchronous context, commands are automatically pipelined due to the nature of an event loop. @@ -406,6 +407,6 @@ as soon as possible in order to prevent allocation of useless memory. ## AUTHORS Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and -Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. +Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. Hiredis is currently maintained by Matt Stancliff (matt at genges dot com) and Jan-Erik Rediger (janerik at fnordig dot com) diff --git a/redis.submodule/deps/hiredis/adapters/libevent.h b/redis.submodule/deps/hiredis/adapters/libevent.h index 273d8b2..a495277 100644 --- a/redis.submodule/deps/hiredis/adapters/libevent.h +++ b/redis.submodule/deps/hiredis/adapters/libevent.h @@ -34,48 +34,113 @@ #include "../hiredis.h" #include "../async.h" +#define REDIS_LIBEVENT_DELETED 0x01 +#define REDIS_LIBEVENT_ENTERED 0x02 + typedef struct redisLibeventEvents { redisAsyncContext *context; - struct event *rev, *wev; + struct event *ev; + struct event_base *base; + struct timeval tv; + short flags; + short state; } redisLibeventEvents; -static void redisLibeventReadEvent(int fd, short event, void *arg) { - ((void)fd); ((void)event); - redisLibeventEvents *e = (redisLibeventEvents*)arg; - redisAsyncHandleRead(e->context); +static void redisLibeventDestroy(redisLibeventEvents *e) { + free(e); } -static void redisLibeventWriteEvent(int fd, short event, void *arg) { - ((void)fd); ((void)event); +static void redisLibeventHandler(int fd, short event, void *arg) { + ((void)fd); redisLibeventEvents *e = (redisLibeventEvents*)arg; - redisAsyncHandleWrite(e->context); + e->state |= REDIS_LIBEVENT_ENTERED; + + #define CHECK_DELETED() if (e->state & REDIS_LIBEVENT_DELETED) {\ + redisLibeventDestroy(e);\ + return; \ + } + + if ((event & EV_TIMEOUT) && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleTimeout(e->context); + CHECK_DELETED(); + } + + if ((event & EV_READ) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleRead(e->context); + CHECK_DELETED(); + } + + if ((event & EV_WRITE) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleWrite(e->context); + CHECK_DELETED(); + } + + e->state &= ~REDIS_LIBEVENT_ENTERED; + #undef CHECK_DELETED +} + +static void redisLibeventUpdate(void *privdata, short flag, int isRemove) { + redisLibeventEvents *e = (redisLibeventEvents *)privdata; + const struct timeval *tv = e->tv.tv_sec || e->tv.tv_usec ? &e->tv : NULL; + + if (isRemove) { + if ((e->flags & flag) == 0) { + return; + } else { + e->flags &= ~flag; + } + } else { + if (e->flags & flag) { + return; + } else { + e->flags |= flag; + } + } + + event_del(e->ev); + event_assign(e->ev, e->base, e->context->c.fd, e->flags | EV_PERSIST, + redisLibeventHandler, privdata); + event_add(e->ev, tv); } static void redisLibeventAddRead(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(e->rev,NULL); + redisLibeventUpdate(privdata, EV_READ, 0); } static void redisLibeventDelRead(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(e->rev); + redisLibeventUpdate(privdata, EV_READ, 1); } static void redisLibeventAddWrite(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(e->wev,NULL); + redisLibeventUpdate(privdata, EV_WRITE, 0); } static void redisLibeventDelWrite(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(e->wev); + redisLibeventUpdate(privdata, EV_WRITE, 1); } static void redisLibeventCleanup(void *privdata) { redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(e->rev); - event_del(e->wev); - free(e); + if (!e) { + return; + } + event_del(e->ev); + event_free(e->ev); + e->ev = NULL; + + if (e->state & REDIS_LIBEVENT_ENTERED) { + e->state |= REDIS_LIBEVENT_DELETED; + } else { + redisLibeventDestroy(e); + } +} + +static void redisLibeventSetTimeout(void *privdata, struct timeval tv) { + redisLibeventEvents *e = (redisLibeventEvents *)privdata; + short flags = e->flags; + e->flags = 0; + e->tv = tv; + redisLibeventUpdate(e, flags, 0); } static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { @@ -87,7 +152,7 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisLibeventEvents*)malloc(sizeof(*e)); + e = (redisLibeventEvents*)calloc(1, sizeof(*e)); e->context = ac; /* Register functions to start/stop listening for events */ @@ -96,13 +161,12 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { ac->ev.addWrite = redisLibeventAddWrite; ac->ev.delWrite = redisLibeventDelWrite; ac->ev.cleanup = redisLibeventCleanup; + ac->ev.scheduleTimer = redisLibeventSetTimeout; ac->ev.data = e; /* Initialize and install read/write events */ - e->rev = event_new(base, c->fd, EV_READ, redisLibeventReadEvent, e); - e->wev = event_new(base, c->fd, EV_WRITE, redisLibeventWriteEvent, e); - event_add(e->rev, NULL); - event_add(e->wev, NULL); + e->ev = event_new(base, c->fd, EV_READ | EV_WRITE, redisLibeventHandler, e); + e->base = base; return REDIS_OK; } #endif diff --git a/redis.submodule/deps/hiredis/adapters/libuv.h b/redis.submodule/deps/hiredis/adapters/libuv.h index ff08c25..39ef7cf 100644 --- a/redis.submodule/deps/hiredis/adapters/libuv.h +++ b/redis.submodule/deps/hiredis/adapters/libuv.h @@ -15,15 +15,12 @@ typedef struct redisLibuvEvents { static void redisLibuvPoll(uv_poll_t* handle, int status, int events) { redisLibuvEvents* p = (redisLibuvEvents*)handle->data; + int ev = (status ? p->events : events); - if (status != 0) { - return; - } - - if (p->context != NULL && (events & UV_READABLE)) { + if (p->context != NULL && (ev & UV_READABLE)) { redisAsyncHandleRead(p->context); } - if (p->context != NULL && (events & UV_WRITABLE)) { + if (p->context != NULL && (ev & UV_WRITABLE)) { redisAsyncHandleWrite(p->context); } } diff --git a/redis.submodule/deps/hiredis/appveyor.yml b/redis.submodule/deps/hiredis/appveyor.yml index 06bbef1..5b43fdb 100644 --- a/redis.submodule/deps/hiredis/appveyor.yml +++ b/redis.submodule/deps/hiredis/appveyor.yml @@ -1,24 +1,14 @@ # Appveyor configuration file for CI build of hiredis on Windows (under Cygwin) environment: matrix: - - CYG_ROOT: C:\cygwin64 - CYG_SETUP: setup-x86_64.exe - CYG_MIRROR: http://cygwin.mirror.constant.com - CYG_CACHE: C:\cygwin64\var\cache\setup - CYG_BASH: C:\cygwin64\bin\bash + - CYG_BASH: C:\cygwin64\bin\bash CC: gcc - - CYG_ROOT: C:\cygwin - CYG_SETUP: setup-x86.exe - CYG_MIRROR: http://cygwin.mirror.constant.com - CYG_CACHE: C:\cygwin\var\cache\setup - CYG_BASH: C:\cygwin\bin\bash + - CYG_BASH: C:\cygwin\bin\bash CC: gcc - TARGET: 32bit - TARGET_VARS: 32bit-vars + CFLAGS: -m32 + CXXFLAGS: -m32 + LDFLAGS: -m32 -# Cache Cygwin files to speed up build -cache: - - '%CYG_CACHE%' clone_depth: 1 # Attempt to ensure we don't try to convert line endings to Win32 CRLF as this will cause build to fail @@ -27,10 +17,8 @@ init: # Install needed build dependencies install: - - ps: 'Start-FileDownload "http://cygwin.com/$env:CYG_SETUP" -FileName "$env:CYG_SETUP"' - - '%CYG_SETUP% --quiet-mode --no-shortcuts --only-site --root "%CYG_ROOT%" --site "%CYG_MIRROR%" --local-package-dir "%CYG_CACHE%" --packages automake,bison,gcc-core,libtool,make,gettext-devel,gettext,intltool,pkg-config,clang,llvm > NUL 2>&1' - '%CYG_BASH% -lc "cygcheck -dc cygwin"' build_script: - 'echo building...' - - '%CYG_BASH% -lc "cd $APPVEYOR_BUILD_FOLDER; exec 0 #include +#ifndef _MSC_VER #include +#endif #include #include #include @@ -40,22 +42,9 @@ #include "net.h" #include "dict.c" #include "sds.h" +#include "win32.h" -#define _EL_ADD_READ(ctx) do { \ - if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \ - } while(0) -#define _EL_DEL_READ(ctx) do { \ - if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \ - } while(0) -#define _EL_ADD_WRITE(ctx) do { \ - if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \ - } while(0) -#define _EL_DEL_WRITE(ctx) do { \ - if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \ - } while(0) -#define _EL_CLEANUP(ctx) do { \ - if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ - } while(0); +#include "async_private.h" /* Forward declaration of function in hiredis.c */ int __redisAppendCommand(redisContext *c, const char *cmd, size_t len); @@ -126,6 +115,7 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) { ac->ev.addWrite = NULL; ac->ev.delWrite = NULL; ac->ev.cleanup = NULL; + ac->ev.scheduleTimer = NULL; ac->onConnect = NULL; ac->onDisconnect = NULL; @@ -150,56 +140,52 @@ static void __redisAsyncCopyError(redisAsyncContext *ac) { ac->errstr = c->errstr; } -redisAsyncContext *redisAsyncConnect(const char *ip, int port) { +redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options) { + redisOptions myOptions = *options; redisContext *c; redisAsyncContext *ac; - c = redisConnectNonBlock(ip,port); - if (c == NULL) + myOptions.options |= REDIS_OPT_NONBLOCK; + c = redisConnectWithOptions(&myOptions); + if (c == NULL) { return NULL; - + } ac = redisAsyncInitialize(c); if (ac == NULL) { redisFree(c); return NULL; } - __redisAsyncCopyError(ac); return ac; } +redisAsyncContext *redisAsyncConnect(const char *ip, int port) { + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + return redisAsyncConnectWithOptions(&options); +} + redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr) { - redisContext *c = redisConnectBindNonBlock(ip,port,source_addr); - redisAsyncContext *ac = redisAsyncInitialize(c); - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + return redisAsyncConnectWithOptions(&options); } redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, const char *source_addr) { - redisContext *c = redisConnectBindNonBlockWithReuse(ip,port,source_addr); - redisAsyncContext *ac = redisAsyncInitialize(c); - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.options |= REDIS_OPT_REUSEADDR; + options.endpoint.tcp.source_addr = source_addr; + return redisAsyncConnectWithOptions(&options); } redisAsyncContext *redisAsyncConnectUnix(const char *path) { - redisContext *c; - redisAsyncContext *ac; - - c = redisConnectUnixNonBlock(path); - if (c == NULL) - return NULL; - - ac = redisAsyncInitialize(c); - if (ac == NULL) { - redisFree(c); - return NULL; - } - - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + return redisAsyncConnectWithOptions(&options); } int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn) { @@ -328,7 +314,7 @@ void redisAsyncFree(redisAsyncContext *ac) { } /* Helper function to make the disconnect happen and clean up. */ -static void __redisAsyncDisconnect(redisAsyncContext *ac) { +void __redisAsyncDisconnect(redisAsyncContext *ac) { redisContext *c = &(ac->c); /* Make sure error is accessible if there is any */ @@ -336,16 +322,23 @@ static void __redisAsyncDisconnect(redisAsyncContext *ac) { if (ac->err == 0) { /* For clean disconnects, there should be no pending callbacks. */ - assert(__redisShiftCallback(&ac->replies,NULL) == REDIS_ERR); + int ret = __redisShiftCallback(&ac->replies,NULL); + assert(ret == REDIS_ERR); } else { /* Disconnection is caused by an error, make sure that pending * callbacks cannot call new commands. */ c->flags |= REDIS_DISCONNECTING; } + /* cleanup event library on disconnect. + * this is safe to call multiple times */ + _EL_CLEANUP(ac); + /* For non-clean disconnects, __redisAsyncFree() will execute pending * callbacks with a NULL-reply. */ - __redisAsyncFree(ac); + if (!(c->flags & REDIS_NO_AUTO_FREE)) { + __redisAsyncFree(ac); + } } /* Tries to do a clean disconnect from Redis, meaning it stops new commands @@ -357,6 +350,9 @@ static void __redisAsyncDisconnect(redisAsyncContext *ac) { void redisAsyncDisconnect(redisAsyncContext *ac) { redisContext *c = &(ac->c); c->flags |= REDIS_DISCONNECTING; + + /** unset the auto-free flag here, because disconnect undoes this */ + c->flags &= ~REDIS_NO_AUTO_FREE; if (!(c->flags & REDIS_IN_CALLBACK) && ac->replies.head == NULL) __redisAsyncDisconnect(ac); } @@ -364,6 +360,7 @@ void redisAsyncDisconnect(redisAsyncContext *ac) { static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, redisCallback *dstcb) { redisContext *c = &(ac->c); dict *callbacks; + redisCallback *cb; dictEntry *de; int pvariant; char *stype; @@ -387,16 +384,28 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, sname = sdsnewlen(reply->element[1]->str,reply->element[1]->len); de = dictFind(callbacks,sname); if (de != NULL) { - memcpy(dstcb,dictGetEntryVal(de),sizeof(*dstcb)); + cb = dictGetEntryVal(de); + + /* If this is an subscribe reply decrease pending counter. */ + if (strcasecmp(stype+pvariant,"subscribe") == 0) { + cb->pending_subs -= 1; + } + + memcpy(dstcb,cb,sizeof(*dstcb)); /* If this is an unsubscribe message, remove it. */ if (strcasecmp(stype+pvariant,"unsubscribe") == 0) { - dictDelete(callbacks,sname); + if (cb->pending_subs == 0) + dictDelete(callbacks,sname); /* If this was the last unsubscribe message, revert to * non-subscribe mode. */ assert(reply->element[2]->type == REDIS_REPLY_INTEGER); - if (reply->element[2]->integer == 0) + + /* Unset subscribed flag only when no pipelined pending subscribe. */ + if (reply->element[2]->integer == 0 + && dictSize(ac->sub.channels) == 0 + && dictSize(ac->sub.patterns) == 0) c->flags &= ~REDIS_SUBSCRIBED; } } @@ -410,7 +419,7 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, void redisProcessCallbacks(redisAsyncContext *ac) { redisContext *c = &(ac->c); - redisCallback cb = {NULL, NULL, NULL}; + redisCallback cb = {NULL, NULL, 0, NULL}; void *reply = NULL; int status; @@ -492,22 +501,34 @@ void redisProcessCallbacks(redisAsyncContext *ac) { * write event fires. When connecting was not successful, the connect callback * is called with a REDIS_ERR status and the context is free'd. */ static int __redisAsyncHandleConnect(redisAsyncContext *ac) { + int completed = 0; redisContext *c = &(ac->c); - - if (redisCheckSocketError(c) == REDIS_ERR) { - /* Try again later when connect(2) is still in progress. */ - if (errno == EINPROGRESS) - return REDIS_OK; - - if (ac->onConnect) ac->onConnect(ac,REDIS_ERR); + if (redisCheckConnectDone(c, &completed) == REDIS_ERR) { + /* Error! */ + redisCheckSocketError(c); + if (ac->onConnect) ac->onConnect(ac, REDIS_ERR); __redisAsyncDisconnect(ac); return REDIS_ERR; + } else if (completed == 1) { + /* connected! */ + if (ac->onConnect) ac->onConnect(ac, REDIS_OK); + c->flags |= REDIS_CONNECTED; + return REDIS_OK; + } else { + return REDIS_OK; } +} - /* Mark context as connected. */ - c->flags |= REDIS_CONNECTED; - if (ac->onConnect) ac->onConnect(ac,REDIS_OK); - return REDIS_OK; +void redisAsyncRead(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + + if (redisBufferRead(c) == REDIS_ERR) { + __redisAsyncDisconnect(ac); + } else { + /* Always re-schedule reads */ + _EL_ADD_READ(ac); + redisProcessCallbacks(ac); + } } /* This function should be called when the socket is readable. @@ -525,18 +546,29 @@ void redisAsyncHandleRead(redisAsyncContext *ac) { return; } - if (redisBufferRead(c) == REDIS_ERR) { + c->funcs->async_read(ac); +} + +void redisAsyncWrite(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + int done = 0; + + if (redisBufferWrite(c,&done) == REDIS_ERR) { __redisAsyncDisconnect(ac); } else { - /* Always re-schedule reads */ + /* Continue writing when not done, stop writing otherwise */ + if (!done) + _EL_ADD_WRITE(ac); + else + _EL_DEL_WRITE(ac); + + /* Always schedule reads after writes */ _EL_ADD_READ(ac); - redisProcessCallbacks(ac); } } void redisAsyncHandleWrite(redisAsyncContext *ac) { redisContext *c = &(ac->c); - int done = 0; if (!(c->flags & REDIS_CONNECTED)) { /* Abort connect was not successful. */ @@ -547,18 +579,37 @@ void redisAsyncHandleWrite(redisAsyncContext *ac) { return; } - if (redisBufferWrite(c,&done) == REDIS_ERR) { - __redisAsyncDisconnect(ac); - } else { - /* Continue writing when not done, stop writing otherwise */ - if (!done) - _EL_ADD_WRITE(ac); - else - _EL_DEL_WRITE(ac); + c->funcs->async_write(ac); +} - /* Always schedule reads after writes */ - _EL_ADD_READ(ac); +void __redisSetError(redisContext *c, int type, const char *str); + +void redisAsyncHandleTimeout(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + redisCallback cb; + + if ((c->flags & REDIS_CONNECTED) && ac->replies.head == NULL) { + /* Nothing to do - just an idle timeout */ + return; } + + if (!c->err) { + __redisSetError(c, REDIS_ERR_TIMEOUT, "Timeout"); + } + + if (!(c->flags & REDIS_CONNECTED) && ac->onConnect) { + ac->onConnect(ac, REDIS_ERR); + } + + while (__redisShiftCallback(&ac->replies, &cb) == REDIS_OK) { + __redisRunCallback(ac, &cb, NULL); + } + + /** + * TODO: Don't automatically sever the connection, + * rather, allow to ignore responses before the queue is clear + */ + __redisAsyncDisconnect(ac); } /* Sets a pointer to the first argument and its length starting at p. Returns @@ -583,6 +634,9 @@ static const char *nextArgument(const char *start, const char **str, size_t *len static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) { redisContext *c = &(ac->c); redisCallback cb; + struct dict *cbdict; + dictEntry *de; + redisCallback *existcb; int pvariant, hasnext; const char *cstr, *astr; size_t clen, alen; @@ -596,6 +650,7 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void /* Setup callback */ cb.fn = fn; cb.privdata = privdata; + cb.pending_subs = 1; /* Find out which command will be appended. */ p = nextArgument(cmd,&cstr,&clen); @@ -612,9 +667,18 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void while ((p = nextArgument(p,&astr,&alen)) != NULL) { sname = sdsnewlen(astr,alen); if (pvariant) - ret = dictReplace(ac->sub.patterns,sname,&cb); + cbdict = ac->sub.patterns; else - ret = dictReplace(ac->sub.channels,sname,&cb); + cbdict = ac->sub.channels; + + de = dictFind(cbdict,sname); + + if (de != NULL) { + existcb = dictGetEntryVal(de); + cb.pending_subs = existcb->pending_subs + 1; + } + + ret = dictReplace(cbdict,sname,&cb); if (ret == 0) sdsfree(sname); } @@ -676,6 +740,8 @@ int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *priv int len; int status; len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen); + if (len < 0) + return REDIS_ERR; status = __redisAsyncCommand(ac,fn,privdata,cmd,len); sdsfree(cmd); return status; @@ -685,3 +751,16 @@ int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void int status = __redisAsyncCommand(ac,fn,privdata,cmd,len); return status; } + +void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv) { + if (!ac->c.timeout) { + ac->c.timeout = calloc(1, sizeof(tv)); + } + + if (tv.tv_sec == ac->c.timeout->tv_sec && + tv.tv_usec == ac->c.timeout->tv_usec) { + return; + } + + *ac->c.timeout = tv; +} diff --git a/redis.submodule/deps/hiredis/async.h b/redis.submodule/deps/hiredis/async.h index 59cbf46..4f6b3b7 100644 --- a/redis.submodule/deps/hiredis/async.h +++ b/redis.submodule/deps/hiredis/async.h @@ -45,6 +45,7 @@ typedef void (redisCallbackFn)(struct redisAsyncContext*, void*, void*); typedef struct redisCallback { struct redisCallback *next; /* simple singly linked list */ redisCallbackFn *fn; + int pending_subs; void *privdata; } redisCallback; @@ -56,6 +57,7 @@ typedef struct redisCallbackList { /* Connection callback prototypes */ typedef void (redisDisconnectCallback)(const struct redisAsyncContext*, int status); typedef void (redisConnectCallback)(const struct redisAsyncContext*, int status); +typedef void(redisTimerCallback)(void *timer, void *privdata); /* Context for an async connection to Redis */ typedef struct redisAsyncContext { @@ -80,6 +82,7 @@ typedef struct redisAsyncContext { void (*addWrite)(void *privdata); void (*delWrite)(void *privdata); void (*cleanup)(void *privdata); + void (*scheduleTimer)(void *privdata, struct timeval tv); } ev; /* Called when either the connection is terminated due to an error or per @@ -92,6 +95,10 @@ typedef struct redisAsyncContext { /* Regular command callbacks */ redisCallbackList replies; + /* Address used for connect() */ + struct sockaddr *saddr; + size_t addrlen; + /* Subscription callbacks */ struct { redisCallbackList invalid; @@ -101,6 +108,7 @@ typedef struct redisAsyncContext { } redisAsyncContext; /* Functions that proxy to hiredis */ +redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options); redisAsyncContext *redisAsyncConnect(const char *ip, int port); redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr); redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, @@ -108,12 +116,17 @@ redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, redisAsyncContext *redisAsyncConnectUnix(const char *path); int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn); int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); + +void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv); void redisAsyncDisconnect(redisAsyncContext *ac); void redisAsyncFree(redisAsyncContext *ac); /* Handle read/write events */ void redisAsyncHandleRead(redisAsyncContext *ac); void redisAsyncHandleWrite(redisAsyncContext *ac); +void redisAsyncHandleTimeout(redisAsyncContext *ac); +void redisAsyncRead(redisAsyncContext *ac); +void redisAsyncWrite(redisAsyncContext *ac); /* Command functions for an async context. Write the command to the * output buffer and register the provided callback. */ diff --git a/redis.submodule/deps/hiredis/async_private.h b/redis.submodule/deps/hiredis/async_private.h new file mode 100644 index 0000000..d0133ae --- /dev/null +++ b/redis.submodule/deps/hiredis/async_private.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_ASYNC_PRIVATE_H +#define __HIREDIS_ASYNC_PRIVATE_H + +#define _EL_ADD_READ(ctx) \ + do { \ + refreshTimeout(ctx); \ + if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \ + } while (0) +#define _EL_DEL_READ(ctx) do { \ + if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \ + } while(0) +#define _EL_ADD_WRITE(ctx) \ + do { \ + refreshTimeout(ctx); \ + if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \ + } while (0) +#define _EL_DEL_WRITE(ctx) do { \ + if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \ + } while(0) +#define _EL_CLEANUP(ctx) do { \ + if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ + ctx->ev.cleanup = NULL; \ + } while(0); + +static inline void refreshTimeout(redisAsyncContext *ctx) { + if (ctx->c.timeout && ctx->ev.scheduleTimer && + (ctx->c.timeout->tv_sec || ctx->c.timeout->tv_usec)) { + ctx->ev.scheduleTimer(ctx->ev.data, *ctx->c.timeout); + // } else { + // printf("Not scheduling timer.. (tmo=%p)\n", ctx->c.timeout); + // if (ctx->c.timeout){ + // printf("tv_sec: %u. tv_usec: %u\n", ctx->c.timeout->tv_sec, + // ctx->c.timeout->tv_usec); + // } + } +} + +void __redisAsyncDisconnect(redisAsyncContext *ac); +void redisProcessCallbacks(redisAsyncContext *ac); + +#endif /* __HIREDIS_ASYNC_PRIVATE_H */ diff --git a/redis.submodule/deps/hiredis/examples/CMakeLists.txt b/redis.submodule/deps/hiredis/examples/CMakeLists.txt new file mode 100644 index 0000000..dd3a313 --- /dev/null +++ b/redis.submodule/deps/hiredis/examples/CMakeLists.txt @@ -0,0 +1,46 @@ +INCLUDE(FindPkgConfig) +# Check for GLib + +PKG_CHECK_MODULES(GLIB2 glib-2.0) +if (GLIB2_FOUND) + INCLUDE_DIRECTORIES(${GLIB2_INCLUDE_DIRS}) + LINK_DIRECTORIES(${GLIB2_LIBRARY_DIRS}) + ADD_EXECUTABLE(example-glib example-glib.c) + TARGET_LINK_LIBRARIES(example-glib hiredis ${GLIB2_LIBRARIES}) +ENDIF(GLIB2_FOUND) + +FIND_PATH(LIBEV ev.h + HINTS /usr/local /usr/opt/local + ENV LIBEV_INCLUDE_DIR) + +if (LIBEV) + # Just compile and link with libev + ADD_EXECUTABLE(example-libev example-libev.c) + TARGET_LINK_LIBRARIES(example-libev hiredis ev) +ENDIF() + +FIND_PATH(LIBEVENT event.h) +if (LIBEVENT) + ADD_EXECUTABLE(example-libevent example-libevent) + TARGET_LINK_LIBRARIES(example-libevent hiredis event) +ENDIF() + +FIND_PATH(LIBUV uv.h) +IF (LIBUV) + ADD_EXECUTABLE(example-libuv example-libuv.c) + TARGET_LINK_LIBRARIES(example-libuv hiredis uv) +ENDIF() + +IF (APPLE) + FIND_LIBRARY(CF CoreFoundation) + ADD_EXECUTABLE(example-macosx example-macosx.c) + TARGET_LINK_LIBRARIES(example-macosx hiredis ${CF}) +ENDIF() + +IF (ENABLE_SSL) + ADD_EXECUTABLE(example-ssl example-ssl.c) + TARGET_LINK_LIBRARIES(example-ssl hiredis hiredis_ssl) +ENDIF() + +ADD_EXECUTABLE(example example.c) +TARGET_LINK_LIBRARIES(example hiredis) diff --git a/redis.submodule/deps/hiredis/examples/example-libevent-ssl.c b/redis.submodule/deps/hiredis/examples/example-libevent-ssl.c new file mode 100644 index 0000000..1021113 --- /dev/null +++ b/redis.submodule/deps/hiredis/examples/example-libevent-ssl.c @@ -0,0 +1,73 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Disconnected...\n"); +} + +int main (int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + struct event_base *base = event_base_new(); + if (argc < 5) { + fprintf(stderr, + "Usage: %s [ca]\n", argv[0]); + exit(1); + } + + const char *value = argv[1]; + size_t nvalue = strlen(value); + + const char *hostname = argv[2]; + int port = atoi(argv[3]); + + const char *cert = argv[4]; + const char *certKey = argv[5]; + const char *caCert = argc > 5 ? argv[6] : NULL; + + redisAsyncContext *c = redisAsyncConnect(hostname, port); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + if (redisSecureConnection(&c->c, caCert, cert, certKey, "sni") != REDIS_OK) { + printf("SSL Error!\n"); + exit(1); + } + + redisLibeventAttach(c,base); + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + redisAsyncCommand(c, NULL, NULL, "SET key %b", value, nvalue); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + event_base_dispatch(base); + return 0; +} diff --git a/redis.submodule/deps/hiredis/examples/example-libevent.c b/redis.submodule/deps/hiredis/examples/example-libevent.c index d333c22..1fe71ae 100644 --- a/redis.submodule/deps/hiredis/examples/example-libevent.c +++ b/redis.submodule/deps/hiredis/examples/example-libevent.c @@ -9,7 +9,12 @@ void getCallback(redisAsyncContext *c, void *r, void *privdata) { redisReply *reply = r; - if (reply == NULL) return; + if (reply == NULL) { + if (c->errstr) { + printf("errstr: %s\n", c->errstr); + } + return; + } printf("argv[%s]: %s\n", (char*)privdata, reply->str); /* Disconnect after receiving the reply to GET */ @@ -35,8 +40,14 @@ void disconnectCallback(const redisAsyncContext *c, int status) { int main (int argc, char **argv) { signal(SIGPIPE, SIG_IGN); struct event_base *base = event_base_new(); + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379); + struct timeval tv = {0}; + tv.tv_sec = 1; + options.timeout = &tv; + - redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + redisAsyncContext *c = redisAsyncConnectWithOptions(&options); if (c->err) { /* Let *c leak for now... */ printf("Error: %s\n", c->errstr); diff --git a/redis.submodule/deps/hiredis/examples/example-ssl.c b/redis.submodule/deps/hiredis/examples/example-ssl.c new file mode 100644 index 0000000..81f4648 --- /dev/null +++ b/redis.submodule/deps/hiredis/examples/example-ssl.c @@ -0,0 +1,97 @@ +#include +#include +#include + +#include +#include + +int main(int argc, char **argv) { + unsigned int j; + redisContext *c; + redisReply *reply; + if (argc < 4) { + printf("Usage: %s [ca]\n", argv[0]); + exit(1); + } + const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + int port = atoi(argv[2]); + const char *cert = argv[3]; + const char *key = argv[4]; + const char *ca = argc > 4 ? argv[5] : NULL; + + struct timeval tv = { 1, 500000 }; // 1.5 seconds + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, hostname, port); + options.timeout = &tv; + c = redisConnectWithOptions(&options); + + if (c == NULL || c->err) { + if (c) { + printf("Connection error: %s\n", c->errstr); + redisFree(c); + } else { + printf("Connection error: can't allocate redis context\n"); + } + exit(1); + } + + if (redisSecureConnection(c, ca, cert, key, "sni") != REDIS_OK) { + printf("Couldn't initialize SSL!\n"); + printf("Error: %s\n", c->errstr); + redisFree(c); + exit(1); + } + + /* PING server */ + reply = redisCommand(c,"PING"); + printf("PING: %s\n", reply->str); + freeReplyObject(reply); + + /* Set a key */ + reply = redisCommand(c,"SET %s %s", "foo", "hello world"); + printf("SET: %s\n", reply->str); + freeReplyObject(reply); + + /* Set a key using binary safe API */ + reply = redisCommand(c,"SET %b %b", "bar", (size_t) 3, "hello", (size_t) 5); + printf("SET (binary API): %s\n", reply->str); + freeReplyObject(reply); + + /* Try a GET and two INCR */ + reply = redisCommand(c,"GET foo"); + printf("GET foo: %s\n", reply->str); + freeReplyObject(reply); + + reply = redisCommand(c,"INCR counter"); + printf("INCR counter: %lld\n", reply->integer); + freeReplyObject(reply); + /* again ... */ + reply = redisCommand(c,"INCR counter"); + printf("INCR counter: %lld\n", reply->integer); + freeReplyObject(reply); + + /* Create a list of numbers, from 0 to 9 */ + reply = redisCommand(c,"DEL mylist"); + freeReplyObject(reply); + for (j = 0; j < 10; j++) { + char buf[64]; + + snprintf(buf,64,"%u",j); + reply = redisCommand(c,"LPUSH mylist element-%s", buf); + freeReplyObject(reply); + } + + /* Let's check what we have inside the list */ + reply = redisCommand(c,"LRANGE mylist 0 -1"); + if (reply->type == REDIS_REPLY_ARRAY) { + for (j = 0; j < reply->elements; j++) { + printf("%u) %s\n", j, reply->element[j]->str); + } + } + freeReplyObject(reply); + + /* Disconnects and frees the context */ + redisFree(c); + + return 0; +} diff --git a/redis.submodule/deps/hiredis/examples/example.c b/redis.submodule/deps/hiredis/examples/example.c index 4d494c5..0e93fc8 100644 --- a/redis.submodule/deps/hiredis/examples/example.c +++ b/redis.submodule/deps/hiredis/examples/example.c @@ -5,14 +5,27 @@ #include int main(int argc, char **argv) { - unsigned int j; + unsigned int j, isunix = 0; redisContext *c; redisReply *reply; const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + + if (argc > 2) { + if (*argv[2] == 'u' || *argv[2] == 'U') { + isunix = 1; + /* in this case, host is the path to the unix socket */ + printf("Will connect to unix socket @%s\n", hostname); + } + } + int port = (argc > 2) ? atoi(argv[2]) : 6379; struct timeval timeout = { 1, 500000 }; // 1.5 seconds - c = redisConnectWithTimeout(hostname, port, timeout); + if (isunix) { + c = redisConnectUnixWithTimeout(hostname, timeout); + } else { + c = redisConnectWithTimeout(hostname, port, timeout); + } if (c == NULL || c->err) { if (c) { printf("Connection error: %s\n", c->errstr); diff --git a/redis.submodule/deps/hiredis/fmacros.h b/redis.submodule/deps/hiredis/fmacros.h index 9a56643..3227faa 100644 --- a/redis.submodule/deps/hiredis/fmacros.h +++ b/redis.submodule/deps/hiredis/fmacros.h @@ -1,25 +1,12 @@ #ifndef __HIREDIS_FMACRO_H #define __HIREDIS_FMACRO_H -#if defined(__linux__) -#define _BSD_SOURCE -#define _DEFAULT_SOURCE -#endif - -#if defined(__CYGWIN__) -#include -#endif - -#if defined(__sun__) -#define _POSIX_C_SOURCE 200112L -#else -#if !(defined(__APPLE__) && defined(__MACH__)) && !(defined(__FreeBSD__)) #define _XOPEN_SOURCE 600 -#endif -#endif +#define _POSIX_C_SOURCE 200112L #if defined(__APPLE__) && defined(__MACH__) -#define _OSX +/* Enable TCP_KEEPALIVE */ +#define _DARWIN_C_SOURCE #endif #endif diff --git a/redis.submodule/deps/hiredis/hiredis.c b/redis.submodule/deps/hiredis/hiredis.c index 18bdfc9..abd94c0 100644 --- a/redis.submodule/deps/hiredis/hiredis.c +++ b/redis.submodule/deps/hiredis/hiredis.c @@ -34,7 +34,6 @@ #include "fmacros.h" #include #include -#include #include #include #include @@ -42,12 +41,24 @@ #include "hiredis.h" #include "net.h" #include "sds.h" +#include "async.h" +#include "win32.h" + +static redisContextFuncs redisContextDefaultFuncs = { + .free_privdata = NULL, + .async_read = redisAsyncRead, + .async_write = redisAsyncWrite, + .read = redisNetRead, + .write = redisNetWrite +}; static redisReply *createReplyObject(int type); static void *createStringObject(const redisReadTask *task, char *str, size_t len); -static void *createArrayObject(const redisReadTask *task, int elements); +static void *createArrayObject(const redisReadTask *task, size_t elements); static void *createIntegerObject(const redisReadTask *task, long long value); +static void *createDoubleObject(const redisReadTask *task, double value, char *str, size_t len); static void *createNilObject(const redisReadTask *task); +static void *createBoolObject(const redisReadTask *task, int bval); /* Default set of functions to build the reply. Keep in mind that such a * function returning NULL is interpreted as OOM. */ @@ -55,7 +66,9 @@ static redisReplyObjectFunctions defaultFunctions = { createStringObject, createArrayObject, createIntegerObject, + createDoubleObject, createNilObject, + createBoolObject, freeReplyObject }; @@ -82,18 +95,19 @@ void freeReplyObject(void *reply) { case REDIS_REPLY_INTEGER: break; /* Nothing to free */ case REDIS_REPLY_ARRAY: + case REDIS_REPLY_MAP: + case REDIS_REPLY_SET: if (r->element != NULL) { for (j = 0; j < r->elements; j++) - if (r->element[j] != NULL) - freeReplyObject(r->element[j]); + freeReplyObject(r->element[j]); free(r->element); } break; case REDIS_REPLY_ERROR: case REDIS_REPLY_STATUS: case REDIS_REPLY_STRING: - if (r->str != NULL) - free(r->str); + case REDIS_REPLY_DOUBLE: + free(r->str); break; } free(r); @@ -107,34 +121,49 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len if (r == NULL) return NULL; - buf = malloc(len+1); - if (buf == NULL) { - freeReplyObject(r); - return NULL; - } - assert(task->type == REDIS_REPLY_ERROR || task->type == REDIS_REPLY_STATUS || - task->type == REDIS_REPLY_STRING); + task->type == REDIS_REPLY_STRING || + task->type == REDIS_REPLY_VERB); /* Copy string value */ - memcpy(buf,str,len); - buf[len] = '\0'; + if (task->type == REDIS_REPLY_VERB) { + buf = malloc(len-4+1); /* Skip 4 bytes of verbatim type header. */ + if (buf == NULL) { + freeReplyObject(r); + return NULL; + } + memcpy(r->vtype,str,3); + r->vtype[3] = '\0'; + memcpy(buf,str+4,len-4); + buf[len-4] = '\0'; + r->len = len-4; + } else { + buf = malloc(len+1); + if (buf == NULL) { + freeReplyObject(r); + return NULL; + } + memcpy(buf,str,len); + buf[len] = '\0'; + r->len = len; + } r->str = buf; - r->len = len; if (task->parent) { parent = task->parent->obj; - assert(parent->type == REDIS_REPLY_ARRAY); + assert(parent->type == REDIS_REPLY_ARRAY || + parent->type == REDIS_REPLY_MAP || + parent->type == REDIS_REPLY_SET); parent->element[task->idx] = r; } return r; } -static void *createArrayObject(const redisReadTask *task, int elements) { +static void *createArrayObject(const redisReadTask *task, size_t elements) { redisReply *r, *parent; - r = createReplyObject(REDIS_REPLY_ARRAY); + r = createReplyObject(task->type); if (r == NULL) return NULL; @@ -150,7 +179,9 @@ static void *createArrayObject(const redisReadTask *task, int elements) { if (task->parent) { parent = task->parent->obj; - assert(parent->type == REDIS_REPLY_ARRAY); + assert(parent->type == REDIS_REPLY_ARRAY || + parent->type == REDIS_REPLY_MAP || + parent->type == REDIS_REPLY_SET); parent->element[task->idx] = r; } return r; @@ -167,7 +198,41 @@ static void *createIntegerObject(const redisReadTask *task, long long value) { if (task->parent) { parent = task->parent->obj; - assert(parent->type == REDIS_REPLY_ARRAY); + assert(parent->type == REDIS_REPLY_ARRAY || + parent->type == REDIS_REPLY_MAP || + parent->type == REDIS_REPLY_SET); + parent->element[task->idx] = r; + } + return r; +} + +static void *createDoubleObject(const redisReadTask *task, double value, char *str, size_t len) { + redisReply *r, *parent; + + r = createReplyObject(REDIS_REPLY_DOUBLE); + if (r == NULL) + return NULL; + + r->dval = value; + r->str = malloc(len+1); + if (r->str == NULL) { + freeReplyObject(r); + return NULL; + } + + /* The double reply also has the original protocol string representing a + * double as a null terminated string. This way the caller does not need + * to format back for string conversion, especially since Redis does efforts + * to make the string more human readable avoiding the calssical double + * decimal string conversion artifacts. */ + memcpy(r->str, str, len); + r->str[len] = '\0'; + + if (task->parent) { + parent = task->parent->obj; + assert(parent->type == REDIS_REPLY_ARRAY || + parent->type == REDIS_REPLY_MAP || + parent->type == REDIS_REPLY_SET); parent->element[task->idx] = r; } return r; @@ -182,7 +247,28 @@ static void *createNilObject(const redisReadTask *task) { if (task->parent) { parent = task->parent->obj; - assert(parent->type == REDIS_REPLY_ARRAY); + assert(parent->type == REDIS_REPLY_ARRAY || + parent->type == REDIS_REPLY_MAP || + parent->type == REDIS_REPLY_SET); + parent->element[task->idx] = r; + } + return r; +} + +static void *createBoolObject(const redisReadTask *task, int bval) { + redisReply *r, *parent; + + r = createReplyObject(REDIS_REPLY_BOOL); + if (r == NULL) + return NULL; + + r->integer = bval != 0; + + if (task->parent) { + parent = task->parent->obj; + assert(parent->type == REDIS_REPLY_ARRAY || + parent->type == REDIS_REPLY_MAP || + parent->type == REDIS_REPLY_SET); parent->element[task->idx] = r; } return r; @@ -432,11 +518,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { } sdsfree(curarg); - - /* No need to check cmd since it is the last statement that can fail, - * but do it anyway to be as defensive as possible. */ - if (cmd != NULL) - free(cmd); + free(cmd); return error_type; } @@ -581,7 +663,7 @@ void __redisSetError(redisContext *c, int type, const char *str) { } else { /* Only REDIS_ERR_IO may lack a description! */ assert(type == REDIS_ERR_IO); - __redis_strerror_r(errno, c->errstr, sizeof(c->errstr)); + strerror_r(errno, c->errstr, sizeof(c->errstr)); } } @@ -589,53 +671,48 @@ redisReader *redisReaderCreate(void) { return redisReaderCreateWithFunctions(&defaultFunctions); } -static redisContext *redisContextInit(void) { +static redisContext *redisContextInit(const redisOptions *options) { redisContext *c; - c = calloc(1,sizeof(redisContext)); + c = calloc(1, sizeof(*c)); if (c == NULL) return NULL; - c->err = 0; - c->errstr[0] = '\0'; + c->funcs = &redisContextDefaultFuncs; c->obuf = sdsempty(); c->reader = redisReaderCreate(); - c->tcp.host = NULL; - c->tcp.source_addr = NULL; - c->unix_sock.path = NULL; - c->timeout = NULL; + c->fd = REDIS_INVALID_FD; if (c->obuf == NULL || c->reader == NULL) { redisFree(c); return NULL; } - + (void)options; /* options are used in other functions */ return c; } void redisFree(redisContext *c) { if (c == NULL) return; - if (c->fd > 0) - close(c->fd); - if (c->obuf != NULL) - sdsfree(c->obuf); - if (c->reader != NULL) - redisReaderFree(c->reader); - if (c->tcp.host) - free(c->tcp.host); - if (c->tcp.source_addr) - free(c->tcp.source_addr); - if (c->unix_sock.path) - free(c->unix_sock.path); - if (c->timeout) - free(c->timeout); + redisNetClose(c); + + sdsfree(c->obuf); + redisReaderFree(c->reader); + free(c->tcp.host); + free(c->tcp.source_addr); + free(c->unix_sock.path); + free(c->timeout); + free(c->saddr); + if (c->funcs->free_privdata) { + c->funcs->free_privdata(c->privdata); + } + memset(c, 0xff, sizeof(*c)); free(c); } -int redisFreeKeepFd(redisContext *c) { - int fd = c->fd; - c->fd = -1; +redisFD redisFreeKeepFd(redisContext *c) { + redisFD fd = c->fd; + c->fd = REDIS_INVALID_FD; redisFree(c); return fd; } @@ -644,10 +721,13 @@ int redisReconnect(redisContext *c) { c->err = 0; memset(c->errstr, '\0', strlen(c->errstr)); - if (c->fd > 0) { - close(c->fd); + if (c->privdata && c->funcs->free_privdata) { + c->funcs->free_privdata(c->privdata); + c->privdata = NULL; } + redisNetClose(c); + sdsfree(c->obuf); redisReaderFree(c->reader); @@ -668,108 +748,107 @@ int redisReconnect(redisContext *c) { return REDIS_ERR; } +redisContext *redisConnectWithOptions(const redisOptions *options) { + redisContext *c = redisContextInit(options); + if (c == NULL) { + return NULL; + } + if (!(options->options & REDIS_OPT_NONBLOCK)) { + c->flags |= REDIS_BLOCK; + } + if (options->options & REDIS_OPT_REUSEADDR) { + c->flags |= REDIS_REUSEADDR; + } + if (options->options & REDIS_OPT_NOAUTOFREE) { + c->flags |= REDIS_NO_AUTO_FREE; + } + + if (options->type == REDIS_CONN_TCP) { + redisContextConnectBindTcp(c, options->endpoint.tcp.ip, + options->endpoint.tcp.port, options->timeout, + options->endpoint.tcp.source_addr); + } else if (options->type == REDIS_CONN_UNIX) { + redisContextConnectUnix(c, options->endpoint.unix_socket, + options->timeout); + } else if (options->type == REDIS_CONN_USERFD) { + c->fd = options->endpoint.fd; + c->flags |= REDIS_CONNECTED; + } else { + // Unknown type - FIXME - FREE + return NULL; + } + if (options->timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) { + redisContextSetTimeout(c, *options->timeout); + } + return c; +} + /* Connect to a Redis instance. On error the field error in the returned * context will be set to the return value of the error function. * When no set of reply functions is given, the default set will be used. */ redisContext *redisConnect(const char *ip, int port) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + return redisConnectWithOptions(&options); } redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,&tv); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.timeout = &tv; + return redisConnectWithOptions(&options); } redisContext *redisConnectNonBlock(const char *ip, int port) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags &= ~REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } redisContext *redisConnectBindNonBlock(const char *ip, int port, const char *source_addr) { - redisContext *c = redisContextInit(); - c->flags &= ~REDIS_BLOCK; - redisContextConnectBindTcp(c,ip,port,NULL,source_addr); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, const char *source_addr) { - redisContext *c = redisContextInit(); - c->flags &= ~REDIS_BLOCK; - c->flags |= REDIS_REUSEADDR; - redisContextConnectBindTcp(c,ip,port,NULL,source_addr); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + options.options |= REDIS_OPT_NONBLOCK|REDIS_OPT_REUSEADDR; + return redisConnectWithOptions(&options); } redisContext *redisConnectUnix(const char *path) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectUnix(c,path,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + return redisConnectWithOptions(&options); } redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectUnix(c,path,&tv); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + options.timeout = &tv; + return redisConnectWithOptions(&options); } redisContext *redisConnectUnixNonBlock(const char *path) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags &= ~REDIS_BLOCK; - redisContextConnectUnix(c,path,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } -redisContext *redisConnectFd(int fd) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->fd = fd; - c->flags |= REDIS_BLOCK | REDIS_CONNECTED; - return c; +redisContext *redisConnectFd(redisFD fd) { + redisOptions options = {0}; + options.type = REDIS_CONN_USERFD; + options.endpoint.fd = fd; + return redisConnectWithOptions(&options); } /* Set read/write timeout on a blocking socket. */ @@ -789,7 +868,7 @@ int redisEnableKeepAlive(redisContext *c) { /* Use this function to handle a read event on the descriptor. It will try * and read some bytes from the socket and feed them to the reply parser. * - * After this function is called, you may use redisContextReadReply to + * After this function is called, you may use redisGetReplyFromReader to * see if there is a reply available. */ int redisBufferRead(redisContext *c) { char buf[1024*16]; @@ -799,22 +878,15 @@ int redisBufferRead(redisContext *c) { if (c->err) return REDIS_ERR; - nread = read(c->fd,buf,sizeof(buf)); - if (nread == -1) { - if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { - /* Try again later */ - } else { - __redisSetError(c,REDIS_ERR_IO,NULL); + nread = c->funcs->read(c, buf, sizeof(buf)); + if (nread > 0) { + if (redisReaderFeed(c->reader, buf, nread) != REDIS_OK) { + __redisSetError(c, c->reader->err, c->reader->errstr); return REDIS_ERR; + } else { } - } else if (nread == 0) { - __redisSetError(c,REDIS_ERR_EOF,"Server closed the connection"); + } else if (nread < 0) { return REDIS_ERR; - } else { - if (redisReaderFeed(c->reader,buf,nread) != REDIS_OK) { - __redisSetError(c,c->reader->err,c->reader->errstr); - return REDIS_ERR; - } } return REDIS_OK; } @@ -829,21 +901,15 @@ int redisBufferRead(redisContext *c) { * c->errstr to hold the appropriate error string. */ int redisBufferWrite(redisContext *c, int *done) { - int nwritten; /* Return early when the context has seen an error. */ if (c->err) return REDIS_ERR; if (sdslen(c->obuf) > 0) { - nwritten = write(c->fd,c->obuf,sdslen(c->obuf)); - if (nwritten == -1) { - if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { - /* Try again later */ - } else { - __redisSetError(c,REDIS_ERR_IO,NULL); - return REDIS_ERR; - } + int nwritten = c->funcs->write(c); + if (nwritten < 0) { + return REDIS_ERR; } else if (nwritten > 0) { if (nwritten == (signed)sdslen(c->obuf)) { sdsfree(c->obuf); @@ -1007,9 +1073,8 @@ void *redisvCommand(redisContext *c, const char *format, va_list ap) { void *redisCommand(redisContext *c, const char *format, ...) { va_list ap; - void *reply = NULL; va_start(ap,format); - reply = redisvCommand(c,format,ap); + void *reply = redisvCommand(c,format,ap); va_end(ap); return reply; } diff --git a/redis.submodule/deps/hiredis/hiredis.h b/redis.submodule/deps/hiredis/hiredis.h index 423d5e5..69dc39c 100644 --- a/redis.submodule/deps/hiredis/hiredis.h +++ b/redis.submodule/deps/hiredis/hiredis.h @@ -35,14 +35,18 @@ #define __HIREDIS_H #include "read.h" #include /* for va_list */ +#ifndef _MSC_VER #include /* for struct timeval */ +#else +struct timeval; /* forward declaration */ +#endif #include /* uintXX_t, etc */ #include "sds.h" /* for sds */ #define HIREDIS_MAJOR 0 -#define HIREDIS_MINOR 13 -#define HIREDIS_PATCH 3 -#define HIREDIS_SONAME 0.13 +#define HIREDIS_MINOR 14 +#define HIREDIS_PATCH 0 +#define HIREDIS_SONAME 0.14 /* Connection type can be blocking or non-blocking and is set in the * least significant bit of the flags field in redisContext. */ @@ -74,36 +78,18 @@ /* Flag that is set when we should set SO_REUSEADDR before calling bind() */ #define REDIS_REUSEADDR 0x80 +/** + * Flag that indicates the user does not want the context to + * be automatically freed upon error + */ +#define REDIS_NO_AUTO_FREE 0x200 + #define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */ /* number of times we retry to connect in the case of EADDRNOTAVAIL and * SO_REUSEADDR is being used. */ #define REDIS_CONNECT_RETRIES 10 -/* strerror_r has two completely different prototypes and behaviors - * depending on system issues, so we need to operate on the error buffer - * differently depending on which strerror_r we're using. */ -#ifndef _GNU_SOURCE -/* "regular" POSIX strerror_r that does the right thing. */ -#define __redis_strerror_r(errno, buf, len) \ - do { \ - strerror_r((errno), (buf), (len)); \ - } while (0) -#else -/* "bad" GNU strerror_r we need to clean up after. */ -#define __redis_strerror_r(errno, buf, len) \ - do { \ - char *err_str = strerror_r((errno), (buf), (len)); \ - /* If return value _isn't_ the start of the buffer we passed in, \ - * then GNU strerror_r returned an internal static buffer and we \ - * need to copy the result into our private buffer. */ \ - if (err_str != (buf)) { \ - strncpy((buf), err_str, ((len) - 1)); \ - buf[(len)-1] = '\0'; \ - } \ - } while (0) -#endif - #ifdef __cplusplus extern "C" { #endif @@ -112,8 +98,12 @@ extern "C" { typedef struct redisReply { int type; /* REDIS_REPLY_* */ long long integer; /* The integer when type is REDIS_REPLY_INTEGER */ + double dval; /* The double when type is REDIS_REPLY_DOUBLE */ size_t len; /* Length of string */ - char *str; /* Used for both REDIS_REPLY_ERROR and REDIS_REPLY_STRING */ + char *str; /* Used for REDIS_REPLY_ERROR, REDIS_REPLY_STRING + and REDIS_REPLY_DOUBLE (in additionl to dval). */ + char vtype[4]; /* Used for REDIS_REPLY_VERB, contains the null + terminated 3 character content type, such as "txt". */ size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */ struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */ } redisReply; @@ -133,14 +123,93 @@ void redisFreeSdsCommand(sds cmd); enum redisConnectionType { REDIS_CONN_TCP, - REDIS_CONN_UNIX + REDIS_CONN_UNIX, + REDIS_CONN_USERFD }; +struct redisSsl; + +#define REDIS_OPT_NONBLOCK 0x01 +#define REDIS_OPT_REUSEADDR 0x02 + +/** + * Don't automatically free the async object on a connection failure, + * or other implicit conditions. Only free on an explicit call to disconnect() or free() + */ +#define REDIS_OPT_NOAUTOFREE 0x04 + +/* In Unix systems a file descriptor is a regular signed int, with -1 + * representing an invalid descriptor. In Windows it is a SOCKET + * (32- or 64-bit unsigned integer depending on the architecture), where + * all bits set (~0) is INVALID_SOCKET. */ +#ifndef _WIN32 +typedef int redisFD; +#define REDIS_INVALID_FD -1 +#else +#ifdef _WIN64 +typedef unsigned long long redisFD; /* SOCKET = 64-bit UINT_PTR */ +#else +typedef unsigned long redisFD; /* SOCKET = 32-bit UINT_PTR */ +#endif +#define REDIS_INVALID_FD ((redisFD)(~0)) /* INVALID_SOCKET */ +#endif + +typedef struct { + /* + * the type of connection to use. This also indicates which + * `endpoint` member field to use + */ + int type; + /* bit field of REDIS_OPT_xxx */ + int options; + /* timeout value. if NULL, no timeout is used */ + const struct timeval *timeout; + union { + /** use this field for tcp/ip connections */ + struct { + const char *source_addr; + const char *ip; + int port; + } tcp; + /** use this field for unix domain sockets */ + const char *unix_socket; + /** + * use this field to have hiredis operate an already-open + * file descriptor */ + redisFD fd; + } endpoint; +} redisOptions; + +/** + * Helper macros to initialize options to their specified fields. + */ +#define REDIS_OPTIONS_SET_TCP(opts, ip_, port_) \ + (opts)->type = REDIS_CONN_TCP; \ + (opts)->endpoint.tcp.ip = ip_; \ + (opts)->endpoint.tcp.port = port_; + +#define REDIS_OPTIONS_SET_UNIX(opts, path) \ + (opts)->type = REDIS_CONN_UNIX; \ + (opts)->endpoint.unix_socket = path; + +struct redisAsyncContext; +struct redisContext; + +typedef struct redisContextFuncs { + void (*free_privdata)(void *); + void (*async_read)(struct redisAsyncContext *); + void (*async_write)(struct redisAsyncContext *); + int (*read)(struct redisContext *, char *, size_t); + int (*write)(struct redisContext *); +} redisContextFuncs; + /* Context for a connection to Redis */ typedef struct redisContext { + const redisContextFuncs *funcs; /* Function table */ + int err; /* Error flags, 0 when there is no error */ char errstr[128]; /* String representation of error when applicable */ - int fd; + redisFD fd; int flags; char *obuf; /* Write buffer */ redisReader *reader; /* Protocol reader */ @@ -158,8 +227,15 @@ typedef struct redisContext { char *path; } unix_sock; + /* For non-blocking connect */ + struct sockadr *saddr; + size_t addrlen; + + /* Additional private data for hiredis addons such as SSL */ + void *privdata; } redisContext; +redisContext *redisConnectWithOptions(const redisOptions *options); redisContext *redisConnect(const char *ip, int port); redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv); redisContext *redisConnectNonBlock(const char *ip, int port); @@ -170,7 +246,7 @@ redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, redisContext *redisConnectUnix(const char *path); redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv); redisContext *redisConnectUnixNonBlock(const char *path); -redisContext *redisConnectFd(int fd); +redisContext *redisConnectFd(redisFD fd); /** * Reconnect the given context using the saved information. @@ -186,7 +262,7 @@ int redisReconnect(redisContext *c); int redisSetTimeout(redisContext *c, const struct timeval tv); int redisEnableKeepAlive(redisContext *c); void redisFree(redisContext *c); -int redisFreeKeepFd(redisContext *c); +redisFD redisFreeKeepFd(redisContext *c); int redisBufferRead(redisContext *c); int redisBufferWrite(redisContext *c, int *done); diff --git a/redis.submodule/deps/hiredis/hiredis.pc.in b/redis.submodule/deps/hiredis/hiredis.pc.in new file mode 100644 index 0000000..140b040 --- /dev/null +++ b/redis.submodule/deps/hiredis/hiredis.pc.in @@ -0,0 +1,11 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include +pkgincludedir=${includedir}/hiredis + +Name: hiredis +Description: Minimalistic C client library for Redis. +Version: @PROJECT_VERSION@ +Libs: -L${libdir} -lhiredis +Cflags: -I${pkgincludedir} -D_FILE_OFFSET_BITS=64 diff --git a/redis.submodule/deps/hiredis/hiredis_ssl.h b/redis.submodule/deps/hiredis/hiredis_ssl.h new file mode 100644 index 0000000..f844f95 --- /dev/null +++ b/redis.submodule/deps/hiredis/hiredis_ssl.h @@ -0,0 +1,53 @@ + +/* + * Copyright (c) 2019, Redis Labs + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_SSL_H +#define __HIREDIS_SSL_H + +/* This is the underlying struct for SSL in ssl.h, which is not included to + * keep build dependencies short here. + */ +struct ssl_st; + +/** + * Secure the connection using SSL. This should be done before any command is + * executed on the connection. + */ +int redisSecureConnection(redisContext *c, const char *capath, const char *certpath, + const char *keypath, const char *servername); + +/** + * Initiate SSL/TLS negotiation on a provided context. + */ + +int redisInitiateSSL(redisContext *c, struct ssl_st *ssl); + +#endif /* __HIREDIS_SSL_H */ diff --git a/redis.submodule/deps/hiredis/hiredis_ssl.pc.in b/redis.submodule/deps/hiredis/hiredis_ssl.pc.in new file mode 100644 index 0000000..588a978 --- /dev/null +++ b/redis.submodule/deps/hiredis/hiredis_ssl.pc.in @@ -0,0 +1,12 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include +pkgincludedir=${includedir}/hiredis + +Name: hiredis_ssl +Description: SSL Support for hiredis. +Version: @PROJECT_VERSION@ +Requires: hiredis +Libs: -L${libdir} -lhiredis_ssl +Libs.private: -lssl -lcrypto diff --git a/redis.submodule/deps/hiredis/net.c b/redis.submodule/deps/hiredis/net.c index 7d41209..e5f40b0 100644 --- a/redis.submodule/deps/hiredis/net.c +++ b/redis.submodule/deps/hiredis/net.c @@ -34,43 +34,72 @@ #include "fmacros.h" #include -#include -#include -#include -#include -#include -#include -#include #include #include -#include #include #include #include -#include #include #include #include "net.h" #include "sds.h" +#include "sockcompat.h" +#include "win32.h" /* Defined in hiredis.c */ void __redisSetError(redisContext *c, int type, const char *str); -static void redisContextCloseFd(redisContext *c) { - if (c && c->fd >= 0) { +void redisNetClose(redisContext *c) { + if (c && c->fd != REDIS_INVALID_FD) { close(c->fd); - c->fd = -1; + c->fd = REDIS_INVALID_FD; } } +int redisNetRead(redisContext *c, char *buf, size_t bufcap) { + int nread = recv(c->fd, buf, bufcap, 0); + if (nread == -1) { + if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { + /* Try again later */ + return 0; + } else if(errno == ETIMEDOUT && (c->flags & REDIS_BLOCK)) { + /* especially in windows */ + __redisSetError(c, REDIS_ERR_TIMEOUT, "recv timeout"); + return -1; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } else if (nread == 0) { + __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection"); + return -1; + } else { + return nread; + } +} + +int redisNetWrite(redisContext *c) { + int nwritten = send(c->fd, c->obuf, sdslen(c->obuf), 0); + if (nwritten < 0) { + if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { + /* Try again later */ + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } + return nwritten; +} + static void __redisSetErrorFromErrno(redisContext *c, int type, const char *prefix) { + int errorno = errno; /* snprintf() may change errno */ char buf[128] = { 0 }; size_t len = 0; if (prefix != NULL) len = snprintf(buf,sizeof(buf),"%s: ",prefix); - __redis_strerror_r(errno, (char *)(buf + len), sizeof(buf) - len); + strerror_r(errorno, (char *)(buf + len), sizeof(buf) - len); __redisSetError(c,type,buf); } @@ -78,15 +107,15 @@ static int redisSetReuseAddr(redisContext *c) { int on = 1; if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } return REDIS_OK; } static int redisCreateSocket(redisContext *c, int type) { - int s; - if ((s = socket(type, SOCK_STREAM, 0)) == -1) { + redisFD s; + if ((s = socket(type, SOCK_STREAM, 0)) == REDIS_INVALID_FD) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); return REDIS_ERR; } @@ -100,6 +129,7 @@ static int redisCreateSocket(redisContext *c, int type) { } static int redisSetBlocking(redisContext *c, int blocking) { +#ifndef _WIN32 int flags; /* Set the socket nonblocking. @@ -107,7 +137,7 @@ static int redisSetBlocking(redisContext *c, int blocking) { * interrupted by a signal. */ if ((flags = fcntl(c->fd, F_GETFL)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_GETFL)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -118,15 +148,23 @@ static int redisSetBlocking(redisContext *c, int blocking) { if (fcntl(c->fd, F_SETFL, flags) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_SETFL)"); - redisContextCloseFd(c); + redisNetClose(c); + return REDIS_ERR; + } +#else + u_long mode = blocking ? 0 : 1; + if (ioctl(c->fd, FIONBIO, &mode) == -1) { + __redisSetErrorFromErrno(c, REDIS_ERR_IO, "ioctl(FIONBIO)"); + redisNetClose(c); return REDIS_ERR; } +#endif /* _WIN32 */ return REDIS_OK; } int redisKeepAlive(redisContext *c, int interval) { int val = 1; - int fd = c->fd; + redisFD fd = c->fd; if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1){ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); @@ -135,14 +173,13 @@ int redisKeepAlive(redisContext *c, int interval) { val = interval; -#ifdef _OSX +#if defined(__APPLE__) && defined(__MACH__) if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE, &val, sizeof(val)) < 0) { __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); return REDIS_ERR; } #else #if defined(__GLIBC__) && !defined(__FreeBSD_kernel__) - val = interval; if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) { __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); return REDIS_ERR; @@ -170,7 +207,7 @@ static int redisSetTcpNoDelay(redisContext *c) { int yes = 1; if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } return REDIS_OK; @@ -212,28 +249,50 @@ static int redisContextWaitReady(redisContext *c, long msec) { if ((res = poll(wfd, 1, msec)) == -1) { __redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } else if (res == 0) { errno = ETIMEDOUT; __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } - if (redisCheckSocketError(c) != REDIS_OK) + if (redisCheckConnectDone(c, &res) != REDIS_OK || res == 0) { + redisCheckSocketError(c); return REDIS_ERR; + } return REDIS_OK; } __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } +int redisCheckConnectDone(redisContext *c, int *completed) { + int rc = connect(c->fd, (const struct sockaddr *)c->saddr, c->addrlen); + if (rc == 0) { + *completed = 1; + return REDIS_OK; + } + switch (errno) { + case EISCONN: + *completed = 1; + return REDIS_OK; + case EALREADY: + case EINPROGRESS: + case EWOULDBLOCK: + *completed = 0; + return REDIS_OK; + default: + return REDIS_ERR; + } +} + int redisCheckSocketError(redisContext *c) { - int err = 0; + int err = 0, errno_saved = errno; socklen_t errlen = sizeof(err); if (getsockopt(c->fd, SOL_SOCKET, SO_ERROR, &err, &errlen) == -1) { @@ -241,6 +300,10 @@ int redisCheckSocketError(redisContext *c) { return REDIS_ERR; } + if (err == 0) { + err = errno_saved; + } + if (err) { errno = err; __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); @@ -251,11 +314,18 @@ int redisCheckSocketError(redisContext *c) { } int redisContextSetTimeout(redisContext *c, const struct timeval tv) { - if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,&tv,sizeof(tv)) == -1) { + const void *to_ptr = &tv; + size_t to_sz = sizeof(tv); +#ifdef _WIN32 + DWORD timeout_msec = tv.tv_sec * 1000 + tv.tv_usec / 1000; + to_ptr = &timeout_msec; + to_sz = sizeof(timeout_msec); +#endif + if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,to_ptr,to_sz) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)"); return REDIS_ERR; } - if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,&tv,sizeof(tv)) == -1) { + if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,to_ptr,to_sz) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_SNDTIMEO)"); return REDIS_ERR; } @@ -265,7 +335,8 @@ int redisContextSetTimeout(redisContext *c, const struct timeval tv) { static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout, const char *source_addr) { - int s, rv, n; + redisFD s; + int rv, n; char _port[6]; /* strlen("65535"); */ struct addrinfo hints, *servinfo, *bservinfo, *p, *b; int blocking = (c->flags & REDIS_BLOCK); @@ -285,8 +356,7 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, * This is a bit ugly, but atleast it works and doesn't leak memory. **/ if (c->tcp.host != addr) { - if (c->tcp.host) - free(c->tcp.host); + free(c->tcp.host); c->tcp.host = strdup(addr); } @@ -299,8 +369,7 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, memcpy(c->timeout, timeout, sizeof(struct timeval)); } } else { - if (c->timeout) - free(c->timeout); + free(c->timeout); c->timeout = NULL; } @@ -336,7 +405,7 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, } for (p = servinfo; p != NULL; p = p->ai_next) { addrretry: - if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) + if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == REDIS_INVALID_FD) continue; c->fd = s; @@ -356,6 +425,7 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, n = 1; if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char*) &n, sizeof(n)) < 0) { + freeaddrinfo(bservinfo); goto error; } } @@ -374,20 +444,34 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, goto error; } } + + /* For repeat connection */ + free(c->saddr); + c->saddr = malloc(p->ai_addrlen); + memcpy(c->saddr, p->ai_addr, p->ai_addrlen); + c->addrlen = p->ai_addrlen; + if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { if (errno == EHOSTUNREACH) { - redisContextCloseFd(c); + redisNetClose(c); continue; - } else if (errno == EINPROGRESS && !blocking) { - /* This is ok. */ + } else if (errno == EINPROGRESS) { + if (blocking) { + goto wait_for_ready; + } + /* This is ok. + * Note that even when it's in blocking mode, we unset blocking + * for `connect()` + */ } else if (errno == EADDRNOTAVAIL && reuseaddr) { if (++reuses >= REDIS_CONNECT_RETRIES) { goto error; } else { - redisContextCloseFd(c); + redisNetClose(c); goto addrretry; } } else { + wait_for_ready: if (redisContextWaitReady(c,timeout_msec) != REDIS_OK) goto error; } @@ -411,7 +495,10 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, error: rv = REDIS_ERR; end: - freeaddrinfo(servinfo); + if(servinfo) { + freeaddrinfo(servinfo); + } + return rv; // Need to return REDIS_OK if alright } @@ -427,11 +514,12 @@ int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, } int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) { +#ifndef _WIN32 int blocking = (c->flags & REDIS_BLOCK); - struct sockaddr_un sa; + struct sockaddr_un *sa; long timeout_msec = -1; - if (redisCreateSocket(c,AF_LOCAL) < 0) + if (redisCreateSocket(c,AF_UNIX) < 0) return REDIS_ERR; if (redisSetBlocking(c,0) != REDIS_OK) return REDIS_ERR; @@ -448,17 +536,18 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time memcpy(c->timeout, timeout, sizeof(struct timeval)); } } else { - if (c->timeout) - free(c->timeout); + free(c->timeout); c->timeout = NULL; } if (redisContextTimeoutMsec(c,&timeout_msec) != REDIS_OK) return REDIS_ERR; - sa.sun_family = AF_LOCAL; - strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1); - if (connect(c->fd, (struct sockaddr*)&sa, sizeof(sa)) == -1) { + sa = (struct sockaddr_un*)(c->saddr = malloc(sizeof(struct sockaddr_un))); + c->addrlen = sizeof(struct sockaddr_un); + sa->sun_family = AF_UNIX; + strncpy(sa->sun_path, path, sizeof(sa->sun_path) - 1); + if (connect(c->fd, (struct sockaddr*)sa, sizeof(*sa)) == -1) { if (errno == EINPROGRESS && !blocking) { /* This is ok. */ } else { @@ -473,4 +562,10 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time c->flags |= REDIS_CONNECTED; return REDIS_OK; +#else + /* We currently do not support Unix sockets for Windows. */ + /* TODO(m): https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/ */ + errno = EPROTONOSUPPORT; + return REDIS_ERR; +#endif /* _WIN32 */ } diff --git a/redis.submodule/deps/hiredis/net.h b/redis.submodule/deps/hiredis/net.h index 2f1a0bf..a4393c0 100644 --- a/redis.submodule/deps/hiredis/net.h +++ b/redis.submodule/deps/hiredis/net.h @@ -37,9 +37,9 @@ #include "hiredis.h" -#if defined(__sun) -#define AF_LOCAL AF_UNIX -#endif +void redisNetClose(redisContext *c); +int redisNetRead(redisContext *c, char *buf, size_t bufcap); +int redisNetWrite(redisContext *c); int redisCheckSocketError(redisContext *c); int redisContextSetTimeout(redisContext *c, const struct timeval tv); @@ -49,5 +49,6 @@ int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, const char *source_addr); int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout); int redisKeepAlive(redisContext *c, int interval); +int redisCheckConnectDone(redisContext *c, int *completed); #endif diff --git a/redis.submodule/deps/hiredis/read.c b/redis.submodule/deps/hiredis/read.c index 50333b5..b9853ea 100644 --- a/redis.submodule/deps/hiredis/read.c +++ b/redis.submodule/deps/hiredis/read.c @@ -29,19 +29,22 @@ * POSSIBILITY OF SUCH DAMAGE. */ - #include "fmacros.h" #include #include #ifndef _MSC_VER #include +#include #endif #include #include #include +#include +#include #include "read.h" #include "sds.h" +#include "win32.h" static void __redisReaderSetError(redisReader *r, int type, const char *str) { size_t len; @@ -52,11 +55,9 @@ static void __redisReaderSetError(redisReader *r, int type, const char *str) { } /* Clear input buffer on errors. */ - if (r->buf != NULL) { - sdsfree(r->buf); - r->buf = NULL; - r->pos = r->len = 0; - } + sdsfree(r->buf); + r->buf = NULL; + r->pos = r->len = 0; /* Reset task stack. */ r->ridx = -1; @@ -143,33 +144,79 @@ static char *seekNewline(char *s, size_t len) { return NULL; } -/* Read a long long value starting at *s, under the assumption that it will be - * terminated by \r\n. Ambiguously returns -1 for unexpected input. */ -static long long readLongLong(char *s) { - long long v = 0; - int dec, mult = 1; - char c; - - if (*s == '-') { - mult = -1; - s++; - } else if (*s == '+') { - mult = 1; - s++; +/* Convert a string into a long long. Returns REDIS_OK if the string could be + * parsed into a (non-overflowing) long long, REDIS_ERR otherwise. The value + * will be set to the parsed value when appropriate. + * + * Note that this function demands that the string strictly represents + * a long long: no spaces or other characters before or after the string + * representing the number are accepted, nor zeroes at the start if not + * for the string "0" representing the zero number. + * + * Because of its strictness, it is safe to use this function to check if + * you can convert a string into a long long, and obtain back the string + * from the number without any loss in the string representation. */ +static int string2ll(const char *s, size_t slen, long long *value) { + const char *p = s; + size_t plen = 0; + int negative = 0; + unsigned long long v; + + if (plen == slen) + return REDIS_ERR; + + /* Special case: first and only digit is 0. */ + if (slen == 1 && p[0] == '0') { + if (value != NULL) *value = 0; + return REDIS_OK; } - while ((c = *(s++)) != '\r') { - dec = c - '0'; - if (dec >= 0 && dec < 10) { - v *= 10; - v += dec; - } else { - /* Should not happen... */ - return -1; - } + if (p[0] == '-') { + negative = 1; + p++; plen++; + + /* Abort on only a negative sign. */ + if (plen == slen) + return REDIS_ERR; + } + + /* First digit should be 1-9, otherwise the string should just be 0. */ + if (p[0] >= '1' && p[0] <= '9') { + v = p[0]-'0'; + p++; plen++; + } else if (p[0] == '0' && slen == 1) { + *value = 0; + return REDIS_OK; + } else { + return REDIS_ERR; } - return mult*v; + while (plen < slen && p[0] >= '0' && p[0] <= '9') { + if (v > (ULLONG_MAX / 10)) /* Overflow. */ + return REDIS_ERR; + v *= 10; + + if (v > (ULLONG_MAX - (p[0]-'0'))) /* Overflow. */ + return REDIS_ERR; + v += p[0]-'0'; + + p++; plen++; + } + + /* Return if not all bytes were used. */ + if (plen < slen) + return REDIS_ERR; + + if (negative) { + if (v > ((unsigned long long)(-(LLONG_MIN+1))+1)) /* Overflow. */ + return REDIS_ERR; + if (value != NULL) *value = -v; + } else { + if (v > LLONG_MAX) /* Overflow. */ + return REDIS_ERR; + if (value != NULL) *value = v; + } + return REDIS_OK; } static char *readLine(redisReader *r, int *_len) { @@ -198,7 +245,9 @@ static void moveToNextTask(redisReader *r) { cur = &(r->rstack[r->ridx]); prv = &(r->rstack[r->ridx-1]); - assert(prv->type == REDIS_REPLY_ARRAY); + assert(prv->type == REDIS_REPLY_ARRAY || + prv->type == REDIS_REPLY_MAP || + prv->type == REDIS_REPLY_SET); if (cur->idx == prv->elements-1) { r->ridx--; } else { @@ -220,10 +269,58 @@ static int processLineItem(redisReader *r) { if ((p = readLine(r,&len)) != NULL) { if (cur->type == REDIS_REPLY_INTEGER) { - if (r->fn && r->fn->createInteger) - obj = r->fn->createInteger(cur,readLongLong(p)); - else + if (r->fn && r->fn->createInteger) { + long long v; + if (string2ll(p, len, &v) == REDIS_ERR) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Bad integer value"); + return REDIS_ERR; + } + obj = r->fn->createInteger(cur,v); + } else { obj = (void*)REDIS_REPLY_INTEGER; + } + } else if (cur->type == REDIS_REPLY_DOUBLE) { + if (r->fn && r->fn->createDouble) { + char buf[326], *eptr; + double d; + + if ((size_t)len >= sizeof(buf)) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Double value is too large"); + return REDIS_ERR; + } + + memcpy(buf,p,len); + buf[len] = '\0'; + + if (strcasecmp(buf,",inf") == 0) { + d = INFINITY; /* Positive infinite. */ + } else if (strcasecmp(buf,",-inf") == 0) { + d = -INFINITY; /* Nevative infinite. */ + } else { + d = strtod((char*)buf,&eptr); + if (buf[0] == '\0' || eptr[0] != '\0' || isnan(d)) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Bad double value"); + return REDIS_ERR; + } + } + obj = r->fn->createDouble(cur,d,buf,len); + } else { + obj = (void*)REDIS_REPLY_DOUBLE; + } + } else if (cur->type == REDIS_REPLY_NIL) { + if (r->fn && r->fn->createNil) + obj = r->fn->createNil(cur); + else + obj = (void*)REDIS_REPLY_NIL; + } else if (cur->type == REDIS_REPLY_BOOL) { + int bval = p[0] == 't' || p[0] == 'T'; + if (r->fn && r->fn->createBool) + obj = r->fn->createBool(cur,bval); + else + obj = (void*)REDIS_REPLY_BOOL; } else { /* Type will be error or status. */ if (r->fn && r->fn->createString) @@ -250,7 +347,7 @@ static int processBulkItem(redisReader *r) { redisReadTask *cur = &(r->rstack[r->ridx]); void *obj = NULL; char *p, *s; - long len; + long long len; unsigned long bytelen; int success = 0; @@ -259,9 +356,20 @@ static int processBulkItem(redisReader *r) { if (s != NULL) { p = r->buf+r->pos; bytelen = s-(r->buf+r->pos)+2; /* include \r\n */ - len = readLongLong(p); - if (len < 0) { + if (string2ll(p, bytelen - 2, &len) == REDIS_ERR) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Bad bulk string length"); + return REDIS_ERR; + } + + if (len < -1 || (LLONG_MAX > SIZE_MAX && len > (long long)SIZE_MAX)) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Bulk string length out of range"); + return REDIS_ERR; + } + + if (len == -1) { /* The nil object can always be created. */ if (r->fn && r->fn->createNil) obj = r->fn->createNil(cur); @@ -272,10 +380,18 @@ static int processBulkItem(redisReader *r) { /* Only continue when the buffer contains the entire bulk item. */ bytelen += len+2; /* include \r\n */ if (r->pos+bytelen <= r->len) { + if ((cur->type == REDIS_REPLY_VERB && len < 4) || + (cur->type == REDIS_REPLY_VERB && s[5] != ':')) + { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Verbatim string 4 bytes of content type are " + "missing or incorrectly encoded."); + return REDIS_ERR; + } if (r->fn && r->fn->createString) obj = r->fn->createString(cur,s+2,len); else - obj = (void*)REDIS_REPLY_STRING; + obj = (void*)(long)cur->type; success = 1; } } @@ -299,12 +415,13 @@ static int processBulkItem(redisReader *r) { return REDIS_ERR; } -static int processMultiBulkItem(redisReader *r) { +/* Process the array, map and set types. */ +static int processAggregateItem(redisReader *r) { redisReadTask *cur = &(r->rstack[r->ridx]); void *obj; char *p; - long elements; - int root = 0; + long long elements; + int root = 0, len; /* Set error for nested multi bulks with depth > 7 */ if (r->ridx == 8) { @@ -313,10 +430,21 @@ static int processMultiBulkItem(redisReader *r) { return REDIS_ERR; } - if ((p = readLine(r,NULL)) != NULL) { - elements = readLongLong(p); + if ((p = readLine(r,&len)) != NULL) { + if (string2ll(p, len, &elements) == REDIS_ERR) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Bad multi-bulk length"); + return REDIS_ERR; + } + root = (r->ridx == 0); + if (elements < -1 || (LLONG_MAX > SIZE_MAX && elements > SIZE_MAX)) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Multi-bulk length out of range"); + return REDIS_ERR; + } + if (elements == -1) { if (r->fn && r->fn->createNil) obj = r->fn->createNil(cur); @@ -330,10 +458,12 @@ static int processMultiBulkItem(redisReader *r) { moveToNextTask(r); } else { + if (cur->type == REDIS_REPLY_MAP) elements *= 2; + if (r->fn && r->fn->createArray) obj = r->fn->createArray(cur,elements); else - obj = (void*)REDIS_REPLY_ARRAY; + obj = (void*)(long)cur->type; if (obj == NULL) { __redisReaderSetErrorOOM(r); @@ -381,12 +511,30 @@ static int processItem(redisReader *r) { case ':': cur->type = REDIS_REPLY_INTEGER; break; + case ',': + cur->type = REDIS_REPLY_DOUBLE; + break; + case '_': + cur->type = REDIS_REPLY_NIL; + break; case '$': cur->type = REDIS_REPLY_STRING; break; case '*': cur->type = REDIS_REPLY_ARRAY; break; + case '%': + cur->type = REDIS_REPLY_MAP; + break; + case '~': + cur->type = REDIS_REPLY_SET; + break; + case '#': + cur->type = REDIS_REPLY_BOOL; + break; + case '=': + cur->type = REDIS_REPLY_VERB; + break; default: __redisReaderSetErrorProtocolByte(r,*p); return REDIS_ERR; @@ -402,11 +550,17 @@ static int processItem(redisReader *r) { case REDIS_REPLY_ERROR: case REDIS_REPLY_STATUS: case REDIS_REPLY_INTEGER: + case REDIS_REPLY_DOUBLE: + case REDIS_REPLY_NIL: + case REDIS_REPLY_BOOL: return processLineItem(r); case REDIS_REPLY_STRING: + case REDIS_REPLY_VERB: return processBulkItem(r); case REDIS_REPLY_ARRAY: - return processMultiBulkItem(r); + case REDIS_REPLY_MAP: + case REDIS_REPLY_SET: + return processAggregateItem(r); default: assert(NULL); return REDIS_ERR; /* Avoid warning. */ @@ -416,12 +570,10 @@ static int processItem(redisReader *r) { redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn) { redisReader *r; - r = calloc(sizeof(redisReader),1); + r = calloc(1,sizeof(redisReader)); if (r == NULL) return NULL; - r->err = 0; - r->errstr[0] = '\0'; r->fn = fn; r->buf = sdsempty(); r->maxbuf = REDIS_READER_MAX_BUF; @@ -435,10 +587,11 @@ redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn) { } void redisReaderFree(redisReader *r) { + if (r == NULL) + return; if (r->reply != NULL && r->fn && r->fn->freeObject) r->fn->freeObject(r->reply); - if (r->buf != NULL) - sdsfree(r->buf); + sdsfree(r->buf); free(r); } @@ -517,8 +670,11 @@ int redisReaderGetReply(redisReader *r, void **reply) { /* Emit a reply when there is one. */ if (r->ridx == -1) { - if (reply != NULL) + if (reply != NULL) { *reply = r->reply; + } else if (r->reply != NULL && r->fn && r->fn->freeObject) { + r->fn->freeObject(r->reply); + } r->reply = NULL; } return REDIS_OK; diff --git a/redis.submodule/deps/hiredis/read.h b/redis.submodule/deps/hiredis/read.h index 2988aa4..5810531 100644 --- a/redis.submodule/deps/hiredis/read.h +++ b/redis.submodule/deps/hiredis/read.h @@ -45,6 +45,7 @@ #define REDIS_ERR_EOF 3 /* End of file */ #define REDIS_ERR_PROTOCOL 4 /* Protocol error */ #define REDIS_ERR_OOM 5 /* Out of memory */ +#define REDIS_ERR_TIMEOUT 6 /* Timed out */ #define REDIS_ERR_OTHER 2 /* Everything else... */ #define REDIS_REPLY_STRING 1 @@ -53,6 +54,14 @@ #define REDIS_REPLY_NIL 4 #define REDIS_REPLY_STATUS 5 #define REDIS_REPLY_ERROR 6 +#define REDIS_REPLY_DOUBLE 7 +#define REDIS_REPLY_BOOL 8 +#define REDIS_REPLY_MAP 9 +#define REDIS_REPLY_SET 10 +#define REDIS_REPLY_ATTR 11 +#define REDIS_REPLY_PUSH 12 +#define REDIS_REPLY_BIGNUM 13 +#define REDIS_REPLY_VERB 14 #define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */ @@ -71,9 +80,11 @@ typedef struct redisReadTask { typedef struct redisReplyObjectFunctions { void *(*createString)(const redisReadTask*, char*, size_t); - void *(*createArray)(const redisReadTask*, int); + void *(*createArray)(const redisReadTask*, size_t); void *(*createInteger)(const redisReadTask*, long long); + void *(*createDouble)(const redisReadTask*, double, char*, size_t); void *(*createNil)(const redisReadTask*); + void *(*createBool)(const redisReadTask*, int); void (*freeObject)(void*); } redisReplyObjectFunctions; diff --git a/redis.submodule/deps/hiredis/sds.c b/redis.submodule/deps/hiredis/sds.c index 923ffd8..6cf7584 100644 --- a/redis.submodule/deps/hiredis/sds.c +++ b/redis.submodule/deps/hiredis/sds.c @@ -219,7 +219,10 @@ sds sdsMakeRoomFor(sds s, size_t addlen) { hdrlen = sdsHdrSize(type); if (oldtype==type) { newsh = s_realloc(sh, hdrlen+newlen+1); - if (newsh == NULL) return NULL; + if (newsh == NULL) { + s_free(sh); + return NULL; + } s = (char*)newsh+hdrlen; } else { /* Since the header size changes, need to move the string forward, @@ -592,6 +595,7 @@ sds sdscatfmt(sds s, char const *fmt, ...) { /* Make sure there is always space for at least 1 char. */ if (sdsavail(s)==0) { s = sdsMakeRoomFor(s,1); + if (s == NULL) goto fmt_error; } switch(*f) { @@ -605,6 +609,7 @@ sds sdscatfmt(sds s, char const *fmt, ...) { l = (next == 's') ? strlen(str) : sdslen(str); if (sdsavail(s) < l) { s = sdsMakeRoomFor(s,l); + if (s == NULL) goto fmt_error; } memcpy(s+i,str,l); sdsinclen(s,l); @@ -621,6 +626,7 @@ sds sdscatfmt(sds s, char const *fmt, ...) { l = sdsll2str(buf,num); if (sdsavail(s) < l) { s = sdsMakeRoomFor(s,l); + if (s == NULL) goto fmt_error; } memcpy(s+i,buf,l); sdsinclen(s,l); @@ -638,6 +644,7 @@ sds sdscatfmt(sds s, char const *fmt, ...) { l = sdsull2str(buf,unum); if (sdsavail(s) < l) { s = sdsMakeRoomFor(s,l); + if (s == NULL) goto fmt_error; } memcpy(s+i,buf,l); sdsinclen(s,l); @@ -662,6 +669,10 @@ sds sdscatfmt(sds s, char const *fmt, ...) { /* Add null-term */ s[i] = '\0'; return s; + +fmt_error: + va_end(ap); + return NULL; } /* Remove the part of the string from left and from right composed just of @@ -1018,10 +1029,18 @@ sds *sdssplitargs(const char *line, int *argc) { if (*p) p++; } /* add the token to the vector */ - vector = s_realloc(vector,((*argc)+1)*sizeof(char*)); - vector[*argc] = current; - (*argc)++; - current = NULL; + { + char **new_vector = s_realloc(vector,((*argc)+1)*sizeof(char*)); + if (new_vector == NULL) { + s_free(vector); + return NULL; + } + + vector = new_vector; + vector[*argc] = current; + (*argc)++; + current = NULL; + } } else { /* Even on empty input string return something not NULL. */ if (vector == NULL) vector = s_malloc(sizeof(void*)); diff --git a/redis.submodule/deps/hiredis/sds.h b/redis.submodule/deps/hiredis/sds.h index 13be75a..3f9a964 100644 --- a/redis.submodule/deps/hiredis/sds.h +++ b/redis.submodule/deps/hiredis/sds.h @@ -34,6 +34,9 @@ #define __SDS_H #define SDS_MAX_PREALLOC (1024*1024) +#ifdef _MSC_VER +#define __attribute__(x) +#endif #include #include @@ -132,20 +135,20 @@ static inline void sdssetlen(sds s, size_t newlen) { case SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); + *fp = (unsigned char)(SDS_TYPE_5 | (newlen << SDS_TYPE_BITS)); } break; case SDS_TYPE_8: - SDS_HDR(8,s)->len = newlen; + SDS_HDR(8,s)->len = (uint8_t)newlen; break; case SDS_TYPE_16: - SDS_HDR(16,s)->len = newlen; + SDS_HDR(16,s)->len = (uint16_t)newlen; break; case SDS_TYPE_32: - SDS_HDR(32,s)->len = newlen; + SDS_HDR(32,s)->len = (uint32_t)newlen; break; case SDS_TYPE_64: - SDS_HDR(64,s)->len = newlen; + SDS_HDR(64,s)->len = (uint64_t)newlen; break; } } @@ -156,21 +159,21 @@ static inline void sdsinclen(sds s, size_t inc) { case SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - unsigned char newlen = SDS_TYPE_5_LEN(flags)+inc; + unsigned char newlen = SDS_TYPE_5_LEN(flags)+(unsigned char)inc; *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); } break; case SDS_TYPE_8: - SDS_HDR(8,s)->len += inc; + SDS_HDR(8,s)->len += (uint8_t)inc; break; case SDS_TYPE_16: - SDS_HDR(16,s)->len += inc; + SDS_HDR(16,s)->len += (uint16_t)inc; break; case SDS_TYPE_32: - SDS_HDR(32,s)->len += inc; + SDS_HDR(32,s)->len += (uint32_t)inc; break; case SDS_TYPE_64: - SDS_HDR(64,s)->len += inc; + SDS_HDR(64,s)->len += (uint64_t)inc; break; } } @@ -200,16 +203,16 @@ static inline void sdssetalloc(sds s, size_t newlen) { /* Nothing to do, this type has no total allocation info. */ break; case SDS_TYPE_8: - SDS_HDR(8,s)->alloc = newlen; + SDS_HDR(8,s)->alloc = (uint8_t)newlen; break; case SDS_TYPE_16: - SDS_HDR(16,s)->alloc = newlen; + SDS_HDR(16,s)->alloc = (uint16_t)newlen; break; case SDS_TYPE_32: - SDS_HDR(32,s)->alloc = newlen; + SDS_HDR(32,s)->alloc = (uint32_t)newlen; break; case SDS_TYPE_64: - SDS_HDR(64,s)->alloc = newlen; + SDS_HDR(64,s)->alloc = (uint64_t)newlen; break; } } diff --git a/redis.submodule/deps/hiredis/sockcompat.c b/redis.submodule/deps/hiredis/sockcompat.c new file mode 100644 index 0000000..4cc2f41 --- /dev/null +++ b/redis.submodule/deps/hiredis/sockcompat.c @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2019, Marcus Geelnard + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define REDIS_SOCKCOMPAT_IMPLEMENTATION +#include "sockcompat.h" + +#ifdef _WIN32 +static int _wsaErrorToErrno(int err) { + switch (err) { + case WSAEWOULDBLOCK: + return EWOULDBLOCK; + case WSAEINPROGRESS: + return EINPROGRESS; + case WSAEALREADY: + return EALREADY; + case WSAENOTSOCK: + return ENOTSOCK; + case WSAEDESTADDRREQ: + return EDESTADDRREQ; + case WSAEMSGSIZE: + return EMSGSIZE; + case WSAEPROTOTYPE: + return EPROTOTYPE; + case WSAENOPROTOOPT: + return ENOPROTOOPT; + case WSAEPROTONOSUPPORT: + return EPROTONOSUPPORT; + case WSAEOPNOTSUPP: + return EOPNOTSUPP; + case WSAEAFNOSUPPORT: + return EAFNOSUPPORT; + case WSAEADDRINUSE: + return EADDRINUSE; + case WSAEADDRNOTAVAIL: + return EADDRNOTAVAIL; + case WSAENETDOWN: + return ENETDOWN; + case WSAENETUNREACH: + return ENETUNREACH; + case WSAENETRESET: + return ENETRESET; + case WSAECONNABORTED: + return ECONNABORTED; + case WSAECONNRESET: + return ECONNRESET; + case WSAENOBUFS: + return ENOBUFS; + case WSAEISCONN: + return EISCONN; + case WSAENOTCONN: + return ENOTCONN; + case WSAETIMEDOUT: + return ETIMEDOUT; + case WSAECONNREFUSED: + return ECONNREFUSED; + case WSAELOOP: + return ELOOP; + case WSAENAMETOOLONG: + return ENAMETOOLONG; + case WSAEHOSTUNREACH: + return EHOSTUNREACH; + case WSAENOTEMPTY: + return ENOTEMPTY; + default: + /* We just return a generic I/O error if we could not find a relevant error. */ + return EIO; + } +} + +static void _updateErrno(int success) { + errno = success ? 0 : _wsaErrorToErrno(WSAGetLastError()); +} + +static int _initWinsock() { + static int s_initialized = 0; + if (!s_initialized) { + static WSADATA wsadata; + int err = WSAStartup(MAKEWORD(2,2), &wsadata); + if (err != 0) { + errno = _wsaErrorToErrno(err); + return 0; + } + s_initialized = 1; + } + return 1; +} + +int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res) { + /* Note: This function is likely to be called before other functions, so run init here. */ + if (!_initWinsock()) { + return EAI_FAIL; + } + + switch (getaddrinfo(node, service, hints, res)) { + case 0: return 0; + case WSATRY_AGAIN: return EAI_AGAIN; + case WSAEINVAL: return EAI_BADFLAGS; + case WSAEAFNOSUPPORT: return EAI_FAMILY; + case WSA_NOT_ENOUGH_MEMORY: return EAI_MEMORY; + case WSAHOST_NOT_FOUND: return EAI_NONAME; + case WSATYPE_NOT_FOUND: return EAI_SERVICE; + case WSAESOCKTNOSUPPORT: return EAI_SOCKTYPE; + default: return EAI_FAIL; /* Including WSANO_RECOVERY */ + } +} + +const char *win32_gai_strerror(int errcode) { + switch (errcode) { + case 0: errcode = 0; break; + case EAI_AGAIN: errcode = WSATRY_AGAIN; break; + case EAI_BADFLAGS: errcode = WSAEINVAL; break; + case EAI_FAMILY: errcode = WSAEAFNOSUPPORT; break; + case EAI_MEMORY: errcode = WSA_NOT_ENOUGH_MEMORY; break; + case EAI_NONAME: errcode = WSAHOST_NOT_FOUND; break; + case EAI_SERVICE: errcode = WSATYPE_NOT_FOUND; break; + case EAI_SOCKTYPE: errcode = WSAESOCKTNOSUPPORT; break; + default: errcode = WSANO_RECOVERY; break; /* Including EAI_FAIL */ + } + return gai_strerror(errcode); +} + +void win32_freeaddrinfo(struct addrinfo *res) { + freeaddrinfo(res); +} + +SOCKET win32_socket(int domain, int type, int protocol) { + SOCKET s; + + /* Note: This function is likely to be called before other functions, so run init here. */ + if (!_initWinsock()) { + return INVALID_SOCKET; + } + + _updateErrno((s = socket(domain, type, protocol)) != INVALID_SOCKET); + return s; +} + +int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp) { + int ret = ioctlsocket(fd, (long)request, argp); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) { + int ret = bind(sockfd, addr, addrlen); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) { + int ret = connect(sockfd, addr, addrlen); + _updateErrno(ret != SOCKET_ERROR); + + /* For Winsock connect(), the WSAEWOULDBLOCK error means the same thing as + * EINPROGRESS for POSIX connect(), so we do that translation to keep POSIX + * logic consistent. */ + if (errno == EWOULDBLOCK) { + errno = EINPROGRESS; + } + + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen) { + int ret = 0; + if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) { + if (*optlen >= sizeof (struct timeval)) { + struct timeval *tv = optval; + DWORD timeout = 0; + socklen_t dwlen = 0; + ret = getsockopt(sockfd, level, optname, (char *)&timeout, &dwlen); + tv->tv_sec = timeout / 1000; + tv->tv_usec = (timeout * 1000) % 1000000; + } else { + ret = WSAEFAULT; + } + *optlen = sizeof (struct timeval); + } else { + ret = getsockopt(sockfd, level, optname, (char*)optval, optlen); + } + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen) { + int ret = 0; + if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) { + struct timeval *tv = optval; + DWORD timeout = tv->tv_sec * 1000 + tv->tv_usec / 1000; + ret = setsockopt(sockfd, level, optname, (const char*)&timeout, sizeof(DWORD)); + } else { + ret = setsockopt(sockfd, level, optname, (const char*)optval, optlen); + } + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_close(SOCKET fd) { + int ret = closesocket(fd); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags) { + int ret = recv(sockfd, (char*)buf, (int)len, flags); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags) { + int ret = send(sockfd, (const char*)buf, (int)len, flags); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout) { + int ret = WSAPoll(fds, nfds, timeout); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} +#endif /* _WIN32 */ diff --git a/redis.submodule/deps/hiredis/sockcompat.h b/redis.submodule/deps/hiredis/sockcompat.h new file mode 100644 index 0000000..56006c1 --- /dev/null +++ b/redis.submodule/deps/hiredis/sockcompat.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2019, Marcus Geelnard + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SOCKCOMPAT_H +#define __SOCKCOMPAT_H + +#ifndef _WIN32 +/* For POSIX systems we use the standard BSD socket API. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#else +/* For Windows we use winsock. */ +#undef _WIN32_WINNT +#define _WIN32_WINNT 0x0600 /* To get WSAPoll etc. */ +#include +#include +#include + +#ifdef _MSC_VER +typedef signed long ssize_t; +#endif + +/* Emulate the parts of the BSD socket API that we need (override the winsock signatures). */ +int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res); +const char *win32_gai_strerror(int errcode); +void win32_freeaddrinfo(struct addrinfo *res); +SOCKET win32_socket(int domain, int type, int protocol); +int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp); +int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen); +int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen); +int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen); +int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen); +int win32_close(SOCKET fd); +ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags); +ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags); +typedef ULONG nfds_t; +int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout); + +#ifndef REDIS_SOCKCOMPAT_IMPLEMENTATION +#define getaddrinfo(node, service, hints, res) win32_getaddrinfo(node, service, hints, res) +#undef gai_strerror +#define gai_strerror(errcode) win32_gai_strerror(errcode) +#define freeaddrinfo(res) win32_freeaddrinfo(res) +#define socket(domain, type, protocol) win32_socket(domain, type, protocol) +#define ioctl(fd, request, argp) win32_ioctl(fd, request, argp) +#define bind(sockfd, addr, addrlen) win32_bind(sockfd, addr, addrlen) +#define connect(sockfd, addr, addrlen) win32_connect(sockfd, addr, addrlen) +#define getsockopt(sockfd, level, optname, optval, optlen) win32_getsockopt(sockfd, level, optname, optval, optlen) +#define setsockopt(sockfd, level, optname, optval, optlen) win32_setsockopt(sockfd, level, optname, optval, optlen) +#define close(fd) win32_close(fd) +#define recv(sockfd, buf, len, flags) win32_recv(sockfd, buf, len, flags) +#define send(sockfd, buf, len, flags) win32_send(sockfd, buf, len, flags) +#define poll(fds, nfds, timeout) win32_poll(fds, nfds, timeout) +#endif /* REDIS_SOCKCOMPAT_IMPLEMENTATION */ +#endif /* _WIN32 */ + +#endif /* __SOCKCOMPAT_H */ diff --git a/redis.submodule/deps/hiredis/ssl.c b/redis.submodule/deps/hiredis/ssl.c new file mode 100644 index 0000000..78ab9e4 --- /dev/null +++ b/redis.submodule/deps/hiredis/ssl.c @@ -0,0 +1,448 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * Copyright (c) 2019, Redis Labs + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "hiredis.h" +#include "async.h" + +#include +#include +#include +#include + +#include +#include + +#include "async_private.h" + +void __redisSetError(redisContext *c, int type, const char *str); + +/* The SSL context is attached to SSL/TLS connections as a privdata. */ +typedef struct redisSSLContext { + /** + * OpenSSL SSL_CTX; It is optional and will not be set when using + * user-supplied SSL. + */ + SSL_CTX *ssl_ctx; + + /** + * OpenSSL SSL object. + */ + SSL *ssl; + + /** + * SSL_write() requires to be called again with the same arguments it was + * previously called with in the event of an SSL_read/SSL_write situation + */ + size_t lastLen; + + /** Whether the SSL layer requires read (possibly before a write) */ + int wantRead; + + /** + * Whether a write was requested prior to a read. If set, the write() + * should resume whenever a read takes place, if possible + */ + int pendingWrite; +} redisSSLContext; + +/* Forward declaration */ +redisContextFuncs redisContextSSLFuncs; + +#ifdef HIREDIS_SSL_TRACE +/** + * Callback used for debugging + */ +static void sslLogCallback(const SSL *ssl, int where, int ret) { + const char *retstr = ""; + int should_log = 1; + /* Ignore low-level SSL stuff */ + + if (where & SSL_CB_ALERT) { + should_log = 1; + } + if (where == SSL_CB_HANDSHAKE_START || where == SSL_CB_HANDSHAKE_DONE) { + should_log = 1; + } + if ((where & SSL_CB_EXIT) && ret == 0) { + should_log = 1; + } + + if (!should_log) { + return; + } + + retstr = SSL_alert_type_string(ret); + printf("ST(0x%x). %s. R(0x%x)%s\n", where, SSL_state_string_long(ssl), ret, retstr); + + if (where == SSL_CB_HANDSHAKE_DONE) { + printf("Using SSL version %s. Cipher=%s\n", SSL_get_version(ssl), SSL_get_cipher_name(ssl)); + } +} +#endif + +/** + * OpenSSL global initialization and locking handling callbacks. + * Note that this is only required for OpenSSL < 1.1.0. + */ + +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#define HIREDIS_USE_CRYPTO_LOCKS +#endif + +#ifdef HIREDIS_USE_CRYPTO_LOCKS +typedef pthread_mutex_t sslLockType; +static void sslLockInit(sslLockType *l) { + pthread_mutex_init(l, NULL); +} +static void sslLockAcquire(sslLockType *l) { + pthread_mutex_lock(l); +} +static void sslLockRelease(sslLockType *l) { + pthread_mutex_unlock(l); +} +static pthread_mutex_t *ossl_locks; + +static void opensslDoLock(int mode, int lkid, const char *f, int line) { + sslLockType *l = ossl_locks + lkid; + + if (mode & CRYPTO_LOCK) { + sslLockAcquire(l); + } else { + sslLockRelease(l); + } + + (void)f; + (void)line; +} + +static void initOpensslLocks(void) { + unsigned ii, nlocks; + if (CRYPTO_get_locking_callback() != NULL) { + /* Someone already set the callback before us. Don't destroy it! */ + return; + } + nlocks = CRYPTO_num_locks(); + ossl_locks = malloc(sizeof(*ossl_locks) * nlocks); + for (ii = 0; ii < nlocks; ii++) { + sslLockInit(ossl_locks + ii); + } + CRYPTO_set_locking_callback(opensslDoLock); +} +#endif /* HIREDIS_USE_CRYPTO_LOCKS */ + +/** + * SSL Connection initialization. + */ + +static int redisSSLConnect(redisContext *c, SSL_CTX *ssl_ctx, SSL *ssl) { + if (c->privdata) { + __redisSetError(c, REDIS_ERR_OTHER, "redisContext was already associated"); + return REDIS_ERR; + } + c->privdata = calloc(1, sizeof(redisSSLContext)); + + c->funcs = &redisContextSSLFuncs; + redisSSLContext *rssl = c->privdata; + + rssl->ssl_ctx = ssl_ctx; + rssl->ssl = ssl; + + SSL_set_mode(rssl->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); + SSL_set_fd(rssl->ssl, c->fd); + SSL_set_connect_state(rssl->ssl); + + ERR_clear_error(); + int rv = SSL_connect(rssl->ssl); + if (rv == 1) { + return REDIS_OK; + } + + rv = SSL_get_error(rssl->ssl, rv); + if (((c->flags & REDIS_BLOCK) == 0) && + (rv == SSL_ERROR_WANT_READ || rv == SSL_ERROR_WANT_WRITE)) { + return REDIS_OK; + } + + if (c->err == 0) { + char err[512]; + if (rv == SSL_ERROR_SYSCALL) + snprintf(err,sizeof(err)-1,"SSL_connect failed: %s",strerror(errno)); + else { + unsigned long e = ERR_peek_last_error(); + snprintf(err,sizeof(err)-1,"SSL_connect failed: %s", + ERR_reason_error_string(e)); + } + __redisSetError(c, REDIS_ERR_IO, err); + } + return REDIS_ERR; +} + +int redisInitiateSSL(redisContext *c, SSL *ssl) { + return redisSSLConnect(c, NULL, ssl); +} + +int redisSecureConnection(redisContext *c, const char *capath, + const char *certpath, const char *keypath, const char *servername) { + + SSL_CTX *ssl_ctx = NULL; + SSL *ssl = NULL; + + /* Initialize global OpenSSL stuff */ + static int isInit = 0; + if (!isInit) { + isInit = 1; + SSL_library_init(); +#ifdef HIREDIS_USE_CRYPTO_LOCKS + initOpensslLocks(); +#endif + } + + ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + if (!ssl_ctx) { + __redisSetError(c, REDIS_ERR_OTHER, "Failed to create SSL_CTX"); + goto error; + } + +#ifdef HIREDIS_SSL_TRACE + SSL_CTX_set_info_callback(ssl_ctx, sslLogCallback); +#endif + SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); + SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL); + if ((certpath != NULL && keypath == NULL) || (keypath != NULL && certpath == NULL)) { + __redisSetError(c, REDIS_ERR_OTHER, "certpath and keypath must be specified together"); + goto error; + } + + if (capath) { + if (!SSL_CTX_load_verify_locations(ssl_ctx, capath, NULL)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid CA certificate"); + goto error; + } + } + if (certpath) { + if (!SSL_CTX_use_certificate_chain_file(ssl_ctx, certpath)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid client certificate"); + goto error; + } + if (!SSL_CTX_use_PrivateKey_file(ssl_ctx, keypath, SSL_FILETYPE_PEM)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid client key"); + goto error; + } + } + + ssl = SSL_new(ssl_ctx); + if (!ssl) { + __redisSetError(c, REDIS_ERR_OTHER, "Couldn't create new SSL instance"); + goto error; + } + if (servername) { + if (!SSL_set_tlsext_host_name(ssl, servername)) { + __redisSetError(c, REDIS_ERR_OTHER, "Couldn't set server name indication"); + goto error; + } + } + + return redisSSLConnect(c, ssl_ctx, ssl); + +error: + if (ssl) SSL_free(ssl); + if (ssl_ctx) SSL_CTX_free(ssl_ctx); + return REDIS_ERR; +} + +static int maybeCheckWant(redisSSLContext *rssl, int rv) { + /** + * If the error is WANT_READ or WANT_WRITE, the appropriate flags are set + * and true is returned. False is returned otherwise + */ + if (rv == SSL_ERROR_WANT_READ) { + rssl->wantRead = 1; + return 1; + } else if (rv == SSL_ERROR_WANT_WRITE) { + rssl->pendingWrite = 1; + return 1; + } else { + return 0; + } +} + +/** + * Implementation of redisContextFuncs for SSL connections. + */ + +static void redisSSLFreeContext(void *privdata){ + redisSSLContext *rsc = privdata; + + if (!rsc) return; + if (rsc->ssl) { + SSL_free(rsc->ssl); + rsc->ssl = NULL; + } + if (rsc->ssl_ctx) { + SSL_CTX_free(rsc->ssl_ctx); + rsc->ssl_ctx = NULL; + } + free(rsc); +} + +static int redisSSLRead(redisContext *c, char *buf, size_t bufcap) { + redisSSLContext *rssl = c->privdata; + + int nread = SSL_read(rssl->ssl, buf, bufcap); + if (nread > 0) { + return nread; + } else if (nread == 0) { + __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection"); + return -1; + } else { + int err = SSL_get_error(rssl->ssl, nread); + if (c->flags & REDIS_BLOCK) { + /** + * In blocking mode, we should never end up in a situation where + * we get an error without it being an actual error, except + * in the case of EINTR, which can be spuriously received from + * debuggers or whatever. + */ + if (errno == EINTR) { + return 0; + } else { + const char *msg = NULL; + if (errno == EAGAIN) { + msg = "Resource temporarily unavailable"; + } + __redisSetError(c, REDIS_ERR_IO, msg); + return -1; + } + } + + /** + * We can very well get an EWOULDBLOCK/EAGAIN, however + */ + if (maybeCheckWant(rssl, err)) { + return 0; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } +} + +static int redisSSLWrite(redisContext *c) { + redisSSLContext *rssl = c->privdata; + + size_t len = rssl->lastLen ? rssl->lastLen : sdslen(c->obuf); + int rv = SSL_write(rssl->ssl, c->obuf, len); + + if (rv > 0) { + rssl->lastLen = 0; + } else if (rv < 0) { + rssl->lastLen = len; + + int err = SSL_get_error(rssl->ssl, rv); + if ((c->flags & REDIS_BLOCK) == 0 && maybeCheckWant(rssl, err)) { + return 0; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } + return rv; +} + +static void redisSSLAsyncRead(redisAsyncContext *ac) { + int rv; + redisSSLContext *rssl = ac->c.privdata; + redisContext *c = &ac->c; + + rssl->wantRead = 0; + + if (rssl->pendingWrite) { + int done; + + /* This is probably just a write event */ + rssl->pendingWrite = 0; + rv = redisBufferWrite(c, &done); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + return; + } else if (!done) { + _EL_ADD_WRITE(ac); + } + } + + rv = redisBufferRead(c); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + } else { + _EL_ADD_READ(ac); + redisProcessCallbacks(ac); + } +} + +static void redisSSLAsyncWrite(redisAsyncContext *ac) { + int rv, done = 0; + redisSSLContext *rssl = ac->c.privdata; + redisContext *c = &ac->c; + + rssl->pendingWrite = 0; + rv = redisBufferWrite(c, &done); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + return; + } + + if (!done) { + if (rssl->wantRead) { + /* Need to read-before-write */ + rssl->pendingWrite = 1; + _EL_DEL_WRITE(ac); + } else { + /* No extra reads needed, just need to write more */ + _EL_ADD_WRITE(ac); + } + } else { + /* Already done! */ + _EL_DEL_WRITE(ac); + } + + /* Always reschedule a read */ + _EL_ADD_READ(ac); +} + +redisContextFuncs redisContextSSLFuncs = { + .free_privdata = redisSSLFreeContext, + .async_read = redisSSLAsyncRead, + .async_write = redisSSLAsyncWrite, + .read = redisSSLRead, + .write = redisSSLWrite +}; + diff --git a/redis.submodule/deps/hiredis/test.c b/redis.submodule/deps/hiredis/test.c index a23d606..8668e18 100644 --- a/redis.submodule/deps/hiredis/test.c +++ b/redis.submodule/deps/hiredis/test.c @@ -3,7 +3,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -11,12 +13,16 @@ #include #include "hiredis.h" +#ifdef HIREDIS_TEST_SSL +#include "hiredis_ssl.h" +#endif #include "net.h" enum connection_type { CONN_TCP, CONN_UNIX, - CONN_FD + CONN_FD, + CONN_SSL }; struct config { @@ -31,6 +37,14 @@ struct config { struct { const char *path; } unix_sock; + + struct { + const char *host; + int port; + const char *ca_cert; + const char *cert; + const char *key; + } ssl; }; /* The following lines make up our testing "framework" :) */ @@ -91,11 +105,27 @@ static int disconnect(redisContext *c, int keep_fd) { return -1; } -static redisContext *connect(struct config config) { +static void do_ssl_handshake(redisContext *c, struct config config) { +#ifdef HIREDIS_TEST_SSL + redisSecureConnection(c, config.ssl.ca_cert, config.ssl.cert, config.ssl.key, NULL); + if (c->err) { + printf("SSL error: %s\n", c->errstr); + redisFree(c); + exit(1); + } +#else + (void) c; + (void) config; +#endif +} + +static redisContext *do_connect(struct config config) { redisContext *c = NULL; if (config.type == CONN_TCP) { c = redisConnect(config.tcp.host, config.tcp.port); + } else if (config.type == CONN_SSL) { + c = redisConnect(config.ssl.host, config.ssl.port); } else if (config.type == CONN_UNIX) { c = redisConnectUnix(config.unix_sock.path); } else if (config.type == CONN_FD) { @@ -119,9 +149,21 @@ static redisContext *connect(struct config config) { exit(1); } + if (config.type == CONN_SSL) { + do_ssl_handshake(c, config); + } + return select_database(c); } +static void do_reconnect(redisContext *c, struct config config) { + redisReconnect(c); + + if (config.type == CONN_SSL) { + do_ssl_handshake(c, config); + } +} + static void test_format_commands(void) { char *cmd; int len; @@ -248,7 +290,7 @@ static void test_append_formatted_commands(struct config config) { char *cmd; int len; - c = connect(config); + c = do_connect(config); test("Append format command: "); @@ -302,6 +344,82 @@ static void test_reply_reader(void) { strncasecmp(reader->errstr,"No support for",14) == 0); redisReaderFree(reader); + test("Correctly parses LLONG_MAX: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, ":9223372036854775807\r\n",22); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_INTEGER && + ((redisReply*)reply)->integer == LLONG_MAX); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Set error when > LLONG_MAX: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, ":9223372036854775808\r\n",22); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Bad integer value") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Correctly parses LLONG_MIN: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, ":-9223372036854775808\r\n",23); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_INTEGER && + ((redisReply*)reply)->integer == LLONG_MIN); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Set error when < LLONG_MIN: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, ":-9223372036854775809\r\n",23); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Bad integer value") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Set error when array < -1: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, "*-2\r\n+asdf\r\n",12); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Multi-bulk length out of range") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Set error when bulk < -1: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, "$-2\r\nasdf\r\n",11); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Bulk string length out of range") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + +#if LLONG_MAX > SIZE_MAX + test("Set error when array > SIZE_MAX: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, "*9223372036854775807\r\n+asdf\r\n",29); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Multi-bulk length out of range") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Set error when bulk > SIZE_MAX: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, "$9223372036854775807\r\nasdf\r\n",28); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Bulk string length out of range") == 0); + freeReplyObject(reply); + redisReaderFree(reader); +#endif + test("Works with NULL functions for reply: "); reader = redisReaderCreate(); reader->fn = NULL; @@ -356,20 +474,35 @@ static void test_free_null(void) { test_cond(reply == NULL); } +#define HIREDIS_BAD_DOMAIN "idontexist-noreally.com" static void test_blocking_connection_errors(void) { redisContext *c; - - test("Returns error when host cannot be resolved: "); - c = redisConnect((char*)"idontexist.test", 6379); - test_cond(c->err == REDIS_ERR_OTHER && - (strcmp(c->errstr,"Name or service not known") == 0 || - strcmp(c->errstr,"Can't resolve: idontexist.test") == 0 || - strcmp(c->errstr,"nodename nor servname provided, or not known") == 0 || - strcmp(c->errstr,"No address associated with hostname") == 0 || - strcmp(c->errstr,"Temporary failure in name resolution") == 0 || - strcmp(c->errstr,"hostname nor servname provided, or not known") == 0 || - strcmp(c->errstr,"no address associated with name") == 0)); - redisFree(c); + struct addrinfo hints = {.ai_family = AF_INET}; + struct addrinfo *ai_tmp = NULL; + + int rv = getaddrinfo(HIREDIS_BAD_DOMAIN, "6379", &hints, &ai_tmp); + if (rv != 0) { + // Address does *not* exist + test("Returns error when host cannot be resolved: "); + // First see if this domain name *actually* resolves to NXDOMAIN + c = redisConnect(HIREDIS_BAD_DOMAIN, 6379); + test_cond( + c->err == REDIS_ERR_OTHER && + (strcmp(c->errstr, "Name or service not known") == 0 || + strcmp(c->errstr, "Can't resolve: " HIREDIS_BAD_DOMAIN) == 0 || + strcmp(c->errstr, "Name does not resolve") == 0 || + strcmp(c->errstr, + "nodename nor servname provided, or not known") == 0 || + strcmp(c->errstr, "No address associated with hostname") == 0 || + strcmp(c->errstr, "Temporary failure in name resolution") == 0 || + strcmp(c->errstr, + "hostname nor servname provided, or not known") == 0 || + strcmp(c->errstr, "no address associated with name") == 0)); + redisFree(c); + } else { + printf("Skipping NXDOMAIN test. Found evil ISP!\n"); + freeaddrinfo(ai_tmp); + } test("Returns error when the port is not open: "); c = redisConnect((char*)"localhost", 1); @@ -387,7 +520,7 @@ static void test_blocking_connection(struct config config) { redisContext *c; redisReply *reply; - c = connect(config); + c = do_connect(config); test("Is able to deliver commands: "); reply = redisCommand(c,"PING"); @@ -468,7 +601,7 @@ static void test_blocking_connection_timeouts(struct config config) { const char *cmd = "DEBUG SLEEP 3\r\n"; struct timeval tv; - c = connect(config); + c = do_connect(config); test("Successfully completes a command when the timeout is not exceeded: "); reply = redisCommand(c,"SET foo fast"); freeReplyObject(reply); @@ -480,9 +613,10 @@ static void test_blocking_connection_timeouts(struct config config) { freeReplyObject(reply); disconnect(c, 0); - c = connect(config); + c = do_connect(config); test("Does not return a reply when the command times out: "); - s = write(c->fd, cmd, strlen(cmd)); + redisAppendFormattedCommand(c, cmd, strlen(cmd)); + s = c->funcs->write(c); tv.tv_sec = 0; tv.tv_usec = 10000; redisSetTimeout(c, tv); @@ -491,7 +625,7 @@ static void test_blocking_connection_timeouts(struct config config) { freeReplyObject(reply); test("Reconnect properly reconnects after a timeout: "); - redisReconnect(c); + do_reconnect(c, config); reply = redisCommand(c, "PING"); test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); freeReplyObject(reply); @@ -499,7 +633,7 @@ static void test_blocking_connection_timeouts(struct config config) { test("Reconnect properly uses owned parameters: "); config.tcp.host = "foo"; config.unix_sock.path = "foo"; - redisReconnect(c); + do_reconnect(c, config); reply = redisCommand(c, "PING"); test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); freeReplyObject(reply); @@ -514,7 +648,7 @@ static void test_blocking_io_errors(struct config config) { int major, minor; /* Connect to target given by config. */ - c = connect(config); + c = do_connect(config); { /* Find out Redis version to determine the path for the next test */ const char *field = "redis_version:"; @@ -549,7 +683,7 @@ static void test_blocking_io_errors(struct config config) { strcmp(c->errstr,"Server closed the connection") == 0); redisFree(c); - c = connect(config); + c = do_connect(config); test("Returns I/O error on socket timeout: "); struct timeval tv = { 0, 1000 }; assert(redisSetTimeout(c,tv) == REDIS_OK); @@ -583,7 +717,7 @@ static void test_invalid_timeout_errors(struct config config) { } static void test_throughput(struct config config) { - redisContext *c = connect(config); + redisContext *c = do_connect(config); redisReply **replies; int i, num; long long t1, t2; @@ -616,6 +750,17 @@ static void test_throughput(struct config config) { free(replies); printf("\t(%dx LRANGE with 500 elements: %.3fs)\n", num, (t2-t1)/1000000.0); + replies = malloc(sizeof(redisReply*)*num); + t1 = usec(); + for (i = 0; i < num; i++) { + replies[i] = redisCommand(c, "INCRBY incrkey %d", 1000000); + assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_INTEGER); + } + t2 = usec(); + for (i = 0; i < num; i++) freeReplyObject(replies[i]); + free(replies); + printf("\t(%dx INCRBY: %.3fs)\n", num, (t2-t1)/1000000.0); + num = 10000; replies = malloc(sizeof(redisReply*)*num); for (i = 0; i < num; i++) @@ -644,6 +789,19 @@ static void test_throughput(struct config config) { free(replies); printf("\t(%dx LRANGE with 500 elements (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); + replies = malloc(sizeof(redisReply*)*num); + for (i = 0; i < num; i++) + redisAppendCommand(c,"INCRBY incrkey %d", 1000000); + t1 = usec(); + for (i = 0; i < num; i++) { + assert(redisGetReply(c, (void*)&replies[i]) == REDIS_OK); + assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_INTEGER); + } + t2 = usec(); + for (i = 0; i < num; i++) freeReplyObject(replies[i]); + free(replies); + printf("\t(%dx INCRBY (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); + disconnect(c, 0); } @@ -778,6 +936,23 @@ int main(int argc, char **argv) { throughput = 0; } else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) { test_inherit_fd = 0; +#ifdef HIREDIS_TEST_SSL + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-port")) { + argv++; argc--; + cfg.ssl.port = atoi(argv[0]); + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-host")) { + argv++; argc--; + cfg.ssl.host = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-ca-cert")) { + argv++; argc--; + cfg.ssl.ca_cert = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-cert")) { + argv++; argc--; + cfg.ssl.cert = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-key")) { + argv++; argc--; + cfg.ssl.key = argv[0]; +#endif } else { fprintf(stderr, "Invalid argument: %s\n", argv[0]); exit(1); @@ -806,6 +981,20 @@ int main(int argc, char **argv) { test_blocking_io_errors(cfg); if (throughput) test_throughput(cfg); +#ifdef HIREDIS_TEST_SSL + if (cfg.ssl.port && cfg.ssl.host) { + printf("\nTesting against SSL connection (%s:%d):\n", cfg.ssl.host, cfg.ssl.port); + cfg.type = CONN_SSL; + + test_blocking_connection(cfg); + test_blocking_connection_timeouts(cfg); + test_blocking_io_errors(cfg); + test_invalid_timeout_errors(cfg); + test_append_formatted_commands(cfg); + if (throughput) test_throughput(cfg); + } +#endif + if (test_inherit_fd) { printf("\nTesting against inherited fd (%s):\n", cfg.unix_sock.path); cfg.type = CONN_FD; diff --git a/redis.submodule/deps/hiredis/test.sh b/redis.submodule/deps/hiredis/test.sh new file mode 100755 index 0000000..2cab9e6 --- /dev/null +++ b/redis.submodule/deps/hiredis/test.sh @@ -0,0 +1,70 @@ +#!/bin/sh -ue + +REDIS_SERVER=${REDIS_SERVER:-redis-server} +REDIS_PORT=${REDIS_PORT:-56379} +REDIS_SSL_PORT=${REDIS_SSL_PORT:-56443} +TEST_SSL=${TEST_SSL:-0} +SSL_TEST_ARGS= + +tmpdir=$(mktemp -d) +PID_FILE=${tmpdir}/hiredis-test-redis.pid +SOCK_FILE=${tmpdir}/hiredis-test-redis.sock + +if [ "$TEST_SSL" = "1" ]; then + SSL_CA_CERT=${tmpdir}/ca.crt + SSL_CA_KEY=${tmpdir}/ca.key + SSL_CERT=${tmpdir}/redis.crt + SSL_KEY=${tmpdir}/redis.key + + openssl genrsa -out ${tmpdir}/ca.key 4096 + openssl req \ + -x509 -new -nodes -sha256 \ + -key ${SSL_CA_KEY} \ + -days 3650 \ + -subj '/CN=Hiredis Test CA' \ + -out ${SSL_CA_CERT} + openssl genrsa -out ${SSL_KEY} 2048 + openssl req \ + -new -sha256 \ + -key ${SSL_KEY} \ + -subj '/CN=Hiredis Test Cert' | \ + openssl x509 \ + -req -sha256 \ + -CA ${SSL_CA_CERT} \ + -CAkey ${SSL_CA_KEY} \ + -CAserial ${tmpdir}/ca.txt \ + -CAcreateserial \ + -days 365 \ + -out ${SSL_CERT} + + SSL_TEST_ARGS="--ssl-host 127.0.0.1 --ssl-port ${REDIS_SSL_PORT} --ssl-ca-cert ${SSL_CA_CERT} --ssl-cert ${SSL_CERT} --ssl-key ${SSL_KEY}" +fi + +cleanup() { + set +e + kill $(cat ${PID_FILE}) + rm -rf ${tmpdir} +} +trap cleanup INT TERM EXIT + +cat > ${tmpdir}/redis.conf <> ${tmpdir}/redis.conf < /* for struct timeval */ + #ifndef inline #define inline __inline #endif +#ifndef strcasecmp +#define strcasecmp stricmp +#endif + +#ifndef strncasecmp +#define strncasecmp strnicmp +#endif + #ifndef va_copy #define va_copy(d,s) ((d) = (s)) #endif @@ -37,6 +47,10 @@ __inline int c99_snprintf(char* str, size_t size, const char* format, ...) return count; } #endif +#endif /* _MSC_VER */ -#endif -#endif \ No newline at end of file +#ifdef _WIN32 +#define strerror_r(errno,buf,len) strerror_s(buf,len,errno) +#endif /* _WIN32 */ + +#endif /* _WIN32_HELPER_INCLUDE */ diff --git a/redis.submodule/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h b/redis.submodule/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h index 290e5cf..2685802 100644 --- a/redis.submodule/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h +++ b/redis.submodule/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h @@ -216,7 +216,7 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, } JEMALLOC_ALWAYS_INLINE int -iget_defrag_hint(tsdn_t *tsdn, void* ptr, int *bin_util, int *run_util) { +iget_defrag_hint(tsdn_t *tsdn, void* ptr) { int defrag = 0; rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); @@ -232,11 +232,22 @@ iget_defrag_hint(tsdn_t *tsdn, void* ptr, int *bin_util, int *run_util) { malloc_mutex_lock(tsdn, &bin->lock); /* don't bother moving allocations from the slab currently used for new allocations */ if (slab != bin->slabcur) { - const bin_info_t *bin_info = &bin_infos[binind]; - size_t availregs = bin_info->nregs * bin->stats.curslabs; - *bin_util = ((long long)bin->stats.curregs<<16) / availregs; - *run_util = ((long long)(bin_info->nregs - extent_nfree_get(slab))<<16) / bin_info->nregs; - defrag = 1; + int free_in_slab = extent_nfree_get(slab); + if (free_in_slab) { + const bin_info_t *bin_info = &bin_infos[binind]; + int curslabs = bin->stats.curslabs; + size_t curregs = bin->stats.curregs; + if (bin->slabcur) { + /* remove slabcur from the overall utilization */ + curregs -= bin_info->nregs - extent_nfree_get(bin->slabcur); + curslabs -= 1; + } + /* Compare the utilization ratio of the slab in question to the total average, + * to avoid precision lost and division, we do that by extrapolating the usage + * of the slab as if all slabs have the same usage. If this slab is less used + * than the average, we'll prefer to evict the data to hopefully more used ones */ + defrag = (bin_info->nregs - free_in_slab) * curslabs <= curregs; + } } malloc_mutex_unlock(tsdn, &bin->lock); } diff --git a/redis.submodule/deps/jemalloc/src/background_thread.c b/redis.submodule/deps/jemalloc/src/background_thread.c index 3517a3b..457669c 100644 --- a/redis.submodule/deps/jemalloc/src/background_thread.c +++ b/redis.submodule/deps/jemalloc/src/background_thread.c @@ -787,7 +787,13 @@ background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) { nstime_init(&stats->run_interval, 0); for (unsigned i = 0; i < max_background_threads; i++) { background_thread_info_t *info = &background_thread_info[i]; - malloc_mutex_lock(tsdn, &info->mtx); + if (malloc_mutex_trylock(tsdn, &info->mtx)) { + /* + * Each background thread run may take a long time; + * avoid waiting on the stats if the thread is active. + */ + continue; + } if (info->state != background_thread_stopped) { num_runs += info->tot_n_runs; nstime_add(&stats->run_interval, &info->tot_sleep_time); diff --git a/redis.submodule/deps/jemalloc/src/jemalloc.c b/redis.submodule/deps/jemalloc/src/jemalloc.c index 5b936cb..585645a 100644 --- a/redis.submodule/deps/jemalloc/src/jemalloc.c +++ b/redis.submodule/deps/jemalloc/src/jemalloc.c @@ -3326,12 +3326,10 @@ jemalloc_postfork_child(void) { /******************************************************************************/ /* Helps the application decide if a pointer is worth re-allocating in order to reduce fragmentation. - * returns 0 if the allocation is in the currently active run, - * or when it is not causing any frag issue (large or huge bin) - * returns the bin utilization and run utilization both in fixed point 16:16. + * returns 1 if the allocation should be moved, and 0 if the allocation be kept. * If the application decides to re-allocate it should use MALLOCX_TCACHE_NONE when doing so. */ JEMALLOC_EXPORT int JEMALLOC_NOTHROW -get_defrag_hint(void* ptr, int *bin_util, int *run_util) { +get_defrag_hint(void* ptr) { assert(ptr != NULL); - return iget_defrag_hint(TSDN_NULL, ptr, bin_util, run_util); + return iget_defrag_hint(TSDN_NULL, ptr); } diff --git a/redis.submodule/deps/linenoise/README.markdown b/redis.submodule/deps/linenoise/README.markdown index e01642c..1afea2a 100644 --- a/redis.submodule/deps/linenoise/README.markdown +++ b/redis.submodule/deps/linenoise/README.markdown @@ -21,7 +21,7 @@ So what usually happens is either: The result is a pollution of binaries without line editing support. -So I spent more or less two hours doing a reality check resulting in this little library: is it *really* needed for a line editing library to be 20k lines of code? Apparently not, it is possibe to get a very small, zero configuration, trivial to embed library, that solves the problem. Smaller programs will just include this, supporing line editing out of the box. Larger programs may use this little library or just checking with configure if readline/libedit is available and resorting to Linenoise if not. +So I spent more or less two hours doing a reality check resulting in this little library: is it *really* needed for a line editing library to be 20k lines of code? Apparently not, it is possibe to get a very small, zero configuration, trivial to embed library, that solves the problem. Smaller programs will just include this, supporting line editing out of the box. Larger programs may use this little library or just checking with configure if readline/libedit is available and resorting to Linenoise if not. ## Terminals, in 2010. @@ -126,6 +126,24 @@ Linenoise has direct support for persisting the history into an history file. The functions `linenoiseHistorySave` and `linenoiseHistoryLoad` do just that. Both functions return -1 on error and 0 on success. +## Mask mode + +Sometimes it is useful to allow the user to type passwords or other +secrets that should not be displayed. For such situations linenoise supports +a "mask mode" that will just replace the characters the user is typing +with `*` characters, like in the following example: + + $ ./linenoise_example + hello> get mykey + echo: 'get mykey' + hello> /mask + hello> ********* + +You can enable and disable mask mode using the following two functions: + + void linenoiseMaskModeEnable(void); + void linenoiseMaskModeDisable(void); + ## Completion Linenoise supports completion, which is the ability to complete the user @@ -222,3 +240,8 @@ Sometimes you may want to clear the screen as a result of something the user typed. You can do this by calling the following function: void linenoiseClearScreen(void); + +## Related projects + +* [Linenoise NG](https://github.com/arangodb/linenoise-ng) is a fork of Linenoise that aims to add more advanced features like UTF-8 support, Windows support and other features. Uses C++ instead of C as development language. +* [Linenoise-swift](https://github.com/andybest/linenoise-swift) is a reimplementation of Linenoise written in Swift. diff --git a/redis.submodule/deps/linenoise/example.c b/redis.submodule/deps/linenoise/example.c index 3a544d3..74358c3 100644 --- a/redis.submodule/deps/linenoise/example.c +++ b/redis.submodule/deps/linenoise/example.c @@ -55,6 +55,7 @@ int main(int argc, char **argv) { * * The typed string is returned as a malloc() allocated string by * linenoise, so the user needs to free() it. */ + while((line = linenoise("hello> ")) != NULL) { /* Do something with the string. */ if (line[0] != '\0' && line[0] != '/') { @@ -65,6 +66,10 @@ int main(int argc, char **argv) { /* The "/historylen" command will change the history len. */ int len = atoi(line+11); linenoiseHistorySetMaxLen(len); + } else if (!strncmp(line, "/mask", 5)) { + linenoiseMaskModeEnable(); + } else if (!strncmp(line, "/unmask", 7)) { + linenoiseMaskModeDisable(); } else if (line[0] == '/') { printf("Unreconized command: %s\n", line); } diff --git a/redis.submodule/deps/linenoise/linenoise.c b/redis.submodule/deps/linenoise/linenoise.c index fce14a7..cfe51e7 100644 --- a/redis.submodule/deps/linenoise/linenoise.c +++ b/redis.submodule/deps/linenoise/linenoise.c @@ -125,6 +125,7 @@ static linenoiseHintsCallback *hintsCallback = NULL; static linenoiseFreeHintsCallback *freeHintsCallback = NULL; static struct termios orig_termios; /* In order to restore at exit.*/ +static int maskmode = 0; /* Show "***" instead of input. For passwords. */ static int rawmode = 0; /* For atexit() function to check if restore is needed*/ static int mlmode = 0; /* Multi line mode. Default is single line. */ static int atexit_registered = 0; /* Register atexit just 1 time. */ @@ -197,6 +198,19 @@ FILE *lndebug_fp = NULL; /* ======================= Low level terminal handling ====================== */ +/* Enable "mask mode". When it is enabled, instead of the input that + * the user is typing, the terminal will just display a corresponding + * number of asterisks, like "****". This is useful for passwords and other + * secrets that should not be displayed. */ +void linenoiseMaskModeEnable(void) { + maskmode = 1; +} + +/* Disable mask mode. */ +void linenoiseMaskModeDisable(void) { + maskmode = 0; +} + /* Set if to use or not the multi line mode. */ void linenoiseSetMultiLine(int ml) { mlmode = ml; @@ -485,6 +499,8 @@ void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) { if (bold == 1 && color == -1) color = 37; if (color != -1 || bold != 0) snprintf(seq,64,"\033[%d;%d;49m",bold,color); + else + seq[0] = '\0'; abAppend(ab,seq,strlen(seq)); abAppend(ab,hint,hintlen); if (color != -1 || bold != 0) @@ -523,7 +539,11 @@ static void refreshSingleLine(struct linenoiseState *l) { abAppend(&ab,seq,strlen(seq)); /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); - abAppend(&ab,buf,len); + if (maskmode == 1) { + while (len--) abAppend(&ab,"*",1); + } else { + abAppend(&ab,buf,len); + } /* Show hits if any. */ refreshShowHints(&ab,l,plen); /* Erase to right */ @@ -577,7 +597,12 @@ static void refreshMultiLine(struct linenoiseState *l) { /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); - abAppend(&ab,l->buf,l->len); + if (maskmode == 1) { + unsigned int i; + for (i = 0; i < l->len; i++) abAppend(&ab,"*",1); + } else { + abAppend(&ab,l->buf,l->len); + } /* Show hits if any. */ refreshShowHints(&ab,l,plen); @@ -645,7 +670,8 @@ int linenoiseEditInsert(struct linenoiseState *l, char c) { if ((!mlmode && l->plen+l->len < l->cols && !hintsCallback)) { /* Avoid a full update of the line in the * trivial case. */ - if (write(l->ofd,&c,1) == -1) return -1; + char d = (maskmode==1) ? '*' : c; + if (write(l->ofd,&d,1) == -1) return -1; } else { refreshLine(l); } diff --git a/redis.submodule/deps/linenoise/linenoise.h b/redis.submodule/deps/linenoise/linenoise.h index ed20232..6dfee73 100644 --- a/redis.submodule/deps/linenoise/linenoise.h +++ b/redis.submodule/deps/linenoise/linenoise.h @@ -65,6 +65,8 @@ int linenoiseHistoryLoad(const char *filename); void linenoiseClearScreen(void); void linenoiseSetMultiLine(int ml); void linenoisePrintKeyCodes(void); +void linenoiseMaskModeEnable(void); +void linenoiseMaskModeDisable(void); #ifdef __cplusplus } diff --git a/redis.submodule/deps/lua/src/lua_struct.c b/redis.submodule/deps/lua/src/lua_struct.c index 4d5f027..c58c8e7 100644 --- a/redis.submodule/deps/lua/src/lua_struct.c +++ b/redis.submodule/deps/lua/src/lua_struct.c @@ -89,12 +89,14 @@ typedef struct Header { } Header; -static int getnum (const char **fmt, int df) { +static int getnum (lua_State *L, const char **fmt, int df) { if (!isdigit(**fmt)) /* no number? */ return df; /* return default value */ else { int a = 0; do { + if (a > (INT_MAX / 10) || a * 10 > (INT_MAX - (**fmt - '0'))) + luaL_error(L, "integral size overflow"); a = a*10 + *((*fmt)++) - '0'; } while (isdigit(**fmt)); return a; @@ -115,9 +117,9 @@ static size_t optsize (lua_State *L, char opt, const char **fmt) { case 'f': return sizeof(float); case 'd': return sizeof(double); case 'x': return 1; - case 'c': return getnum(fmt, 1); + case 'c': return getnum(L, fmt, 1); case 'i': case 'I': { - int sz = getnum(fmt, sizeof(int)); + int sz = getnum(L, fmt, sizeof(int)); if (sz > MAXINTSIZE) luaL_error(L, "integral size %d is larger than limit of %d", sz, MAXINTSIZE); @@ -150,7 +152,7 @@ static void controloptions (lua_State *L, int opt, const char **fmt, case '>': h->endian = BIG; return; case '<': h->endian = LITTLE; return; case '!': { - int a = getnum(fmt, MAXALIGN); + int a = getnum(L, fmt, MAXALIGN); if (!isp2(a)) luaL_error(L, "alignment %d is not a power of 2", a); h->align = a; diff --git a/redis.submodule/redis.conf b/redis.submodule/redis.conf index d74ff98..1aa7605 100644 --- a/redis.submodule/redis.conf +++ b/redis.submodule/redis.conf @@ -129,6 +129,76 @@ timeout 0 # Redis default starting with Redis 3.2.1. tcp-keepalive 300 +################################# TLS/SSL ##################################### + +# By default, TLS/SSL is disabled. To enable it, the "tls-port" configuration +# directive can be used to define TLS-listening ports. To enable TLS on the +# default port, use: +# +# port 0 +# tls-port 6379 + +# Configure a X.509 certificate and private key to use for authenticating the +# server to connected clients, masters or cluster peers. These files should be +# PEM formatted. +# +# tls-cert-file redis.crt +# tls-key-file redis.key + +# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: +# +# tls-dh-params-file redis.dh + +# Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL +# clients and peers. Redis requires an explicit configuration of at least one +# of these, and will not implicitly use the system wide configuration. +# +# tls-ca-cert-file ca.crt +# tls-ca-cert-dir /etc/ssl/certs + +# By default, clients (including replica servers) on a TLS port are required +# to authenticate using valid client side certificates. +# +# It is possible to disable authentication using this directive. +# +# tls-auth-clients no + +# By default, a Redis replica does not attempt to establish a TLS connection +# with its master. +# +# Use the following directive to enable TLS on replication links. +# +# tls-replication yes + +# By default, the Redis Cluster bus uses a plain TCP connection. To enable +# TLS for the bus protocol, use the following directive: +# +# tls-cluster yes + +# Explicitly specify TLS versions to support. Allowed values are case insensitive +# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) or +# any combination. To enable only TLSv1.2 and TLSv1.3, use: +# +# tls-protocols "TLSv1.2 TLSv1.3" + +# Configure allowed ciphers. See the ciphers(1ssl) manpage for more information +# about the syntax of this string. +# +# Note: this configuration applies only to <= TLSv1.2. +# +# tls-ciphers DEFAULT:!MEDIUM + +# Configure allowed TLSv1.3 ciphersuites. See the ciphers(1ssl) manpage for more +# information about the syntax of this string, and specifically for TLSv1.3 +# ciphersuites. +# +# tls-ciphersuites TLS_CHACHA20_POLY1305_SHA256 + +# When choosing a cipher, use the server's preference instead of the client +# preference. By default, the server follows the client's preference. +# +# tls-prefer-server-ciphers yes + ################################# GENERAL ##################################### # By default Redis does not run as a daemon. Use 'yes' if you need it. @@ -252,6 +322,19 @@ rdbchecksum yes # The filename where to dump the DB dbfilename dump.rdb +# Remove RDB files used by replication in instances without persistence +# enabled. By default this option is disabled, however there are environments +# where for regulations or other security concerns, RDB files persisted on +# disk by masters in order to feed replicas, or stored on disk by replicas +# in order to load them for the initial synchronization, should be deleted +# ASAP. Note that this option ONLY WORKS in instances that have both AOF +# and RDB persistence disabled, otherwise is completely ignored. +# +# An alternative (and sometimes better) way to obtain the same effect is +# to use diskless replication on both master and replicas instances. However +# in the case of replicas, diskless is not always an option. +rdb-del-sync-files no + # The working directory. # # The DB will be written inside this directory, with the filename specified @@ -291,6 +374,17 @@ dir ./ # refuse the replica request. # # masterauth +# +# However this is not enough if you are using Redis ACLs (for Redis version +# 6 or greater), and the default user is not capable of running the PSYNC +# command and/or other commands needed for replication. In this case it's +# better to configure a special user to use with replication, and specify the +# masteruser configuration as such: +# +# masteruser +# +# When masteruser is specified, the replica will authenticate against its +# master using the new AUTH form: AUTH . # When a replica loses its connection with the master, or when the replication # is still in progress, the replica can act in two different ways: @@ -325,13 +419,11 @@ replica-read-only yes # Replication SYNC strategy: disk or socket. # -# ------------------------------------------------------- -# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY -# ------------------------------------------------------- +# New replicas and reconnecting replicas that are not able to continue the +# replication process just receiving differences, need to do what is called a +# "full synchronization". An RDB file is transmitted from the master to the +# replicas. # -# New replicas and reconnecting replicas that are not able to continue the replication -# process just receiving differences, need to do what is called a "full -# synchronization". An RDB file is transmitted from the master to the replicas. # The transmission can happen in two different ways: # # 1) Disk-backed: The Redis master creates a new process that writes the RDB @@ -341,14 +433,14 @@ replica-read-only yes # RDB file to replica sockets, without touching the disk at all. # # With disk-backed replication, while the RDB file is generated, more replicas -# can be queued and served with the RDB file as soon as the current child producing -# the RDB file finishes its work. With diskless replication instead once -# the transfer starts, new replicas arriving will be queued and a new transfer -# will start when the current one terminates. +# can be queued and served with the RDB file as soon as the current child +# producing the RDB file finishes its work. With diskless replication instead +# once the transfer starts, new replicas arriving will be queued and a new +# transfer will start when the current one terminates. # # When diskless replication is used, the master waits a configurable amount of -# time (in seconds) before starting the transfer in the hope that multiple replicas -# will arrive and the transfer can be parallelized. +# time (in seconds) before starting the transfer in the hope that multiple +# replicas will arrive and the transfer can be parallelized. # # With slow disks and fast (large bandwidth) networks, diskless replication # works better. @@ -359,16 +451,42 @@ repl-diskless-sync no # to the replicas. # # This is important since once the transfer starts, it is not possible to serve -# new replicas arriving, that will be queued for the next RDB transfer, so the server -# waits a delay in order to let more replicas arrive. +# new replicas arriving, that will be queued for the next RDB transfer, so the +# server waits a delay in order to let more replicas arrive. # # The delay is specified in seconds, and by default is 5 seconds. To disable # it entirely just set it to 0 seconds and the transfer will start ASAP. repl-diskless-sync-delay 5 -# Replicas send PINGs to server in a predefined interval. It's possible to change -# this interval with the repl_ping_replica_period option. The default value is 10 -# seconds. +# ----------------------------------------------------------------------------- +# WARNING: RDB diskless load is experimental. Since in this setup the replica +# does not immediately store an RDB on disk, it may cause data loss during +# failovers. RDB diskless load + Redis modules not handling I/O reads may also +# cause Redis to abort in case of I/O errors during the initial synchronization +# stage with the master. Use only if your do what you are doing. +# ----------------------------------------------------------------------------- +# +# Replica can load the RDB it reads from the replication link directly from the +# socket, or store the RDB to a file and read that file after it was completely +# recived from the master. +# +# In many cases the disk is slower than the network, and storing and loading +# the RDB file may increase replication time (and even increase the master's +# Copy on Write memory and salve buffers). +# However, parsing the RDB file directly from the socket may mean that we have +# to flush the contents of the current database before the full rdb was +# received. For this reason we have the following options: +# +# "disabled" - Don't use diskless load (store the rdb file to the disk first) +# "on-empty-db" - Use diskless load only when it is completely safe. +# "swapdb" - Keep a copy of the current db contents in RAM while parsing +# the data directly from the socket. note that this requires +# sufficient memory, if you don't have it, you risk an OOM kill. +repl-diskless-load disabled + +# Replicas send PINGs to server in a predefined interval. It's possible to +# change this interval with the repl_ping_replica_period option. The default +# value is 10 seconds. # # repl-ping-replica-period 10 @@ -400,10 +518,10 @@ repl-diskless-sync-delay 5 repl-disable-tcp-nodelay no # Set the replication backlog size. The backlog is a buffer that accumulates -# replica data when replicas are disconnected for some time, so that when a replica -# wants to reconnect again, often a full resync is not needed, but a partial -# resync is enough, just passing the portion of data the replica missed while -# disconnected. +# replica data when replicas are disconnected for some time, so that when a +# replica wants to reconnect again, often a full resync is not needed, but a +# partial resync is enough, just passing the portion of data the replica +# missed while disconnected. # # The bigger the replication backlog, the longer the time the replica can be # disconnected and later be able to perform a partial resynchronization. @@ -425,13 +543,13 @@ repl-disable-tcp-nodelay no # # repl-backlog-ttl 3600 -# The replica priority is an integer number published by Redis in the INFO output. -# It is used by Redis Sentinel in order to select a replica to promote into a -# master if the master is no longer working correctly. +# The replica priority is an integer number published by Redis in the INFO +# output. It is used by Redis Sentinel in order to select a replica to promote +# into a master if the master is no longer working correctly. # # A replica with a low priority number is considered better for promotion, so -# for instance if there are three replicas with priority 10, 100, 25 Sentinel will -# pick the one with priority 10, that is the lowest. +# for instance if there are three replicas with priority 10, 100, 25 Sentinel +# will pick the one with priority 10, that is the lowest. # # However a special priority of 0 marks the replica as not able to perform the # role of master, so a replica with priority of 0 will never be selected by @@ -491,22 +609,174 @@ replica-priority 100 # replica-announce-ip 5.5.5.5 # replica-announce-port 1234 -################################## SECURITY ################################### +############################### KEYS TRACKING ################################# -# Require clients to issue AUTH before processing any other -# commands. This might be useful in environments in which you do not trust -# others with access to the host running redis-server. +# Redis implements server assisted support for client side caching of values. +# This is implemented using an invalidation table that remembers, using +# 16 millions of slots, what clients may have certain subsets of keys. In turn +# this is used in order to send invalidation messages to clients. Please +# to understand more about the feature check this page: # -# This should stay commented out for backward compatibility and because most -# people do not need auth (e.g. they run their own servers). +# https://redis.io/topics/client-side-caching # +# When tracking is enabled for a client, all the read only queries are assumed +# to be cached: this will force Redis to store information in the invalidation +# table. When keys are modified, such information is flushed away, and +# invalidation messages are sent to the clients. However if the workload is +# heavily dominated by reads, Redis could use more and more memory in order +# to track the keys fetched by many clients. +# +# For this reason it is possible to configure a maximum fill value for the +# invalidation table. By default it is set to 1M of keys, and once this limit +# is reached, Redis will start to evict keys in the invalidation table +# even if they were not modified, just to reclaim memory: this will in turn +# force the clients to invalidate the cached values. Basically the table +# maximum size is a trade off between the memory you want to spend server +# side to track information about who cached what, and the ability of clients +# to retain cached objects in memory. +# +# If you set the value to 0, it means there are no limits, and Redis will +# retain as many keys as needed in the invalidation table. +# In the "stats" INFO section, you can find information about the number of +# keys in the invalidation table at every given moment. +# +# Note: when key tracking is used in broadcasting mode, no memory is used +# in the server side so this setting is useless. +# +# tracking-table-max-keys 1000000 + +################################## SECURITY ################################### + # Warning: since Redis is pretty fast an outside user can try up to -# 150k passwords per second against a good box. This means that you should -# use a very strong password otherwise it will be very easy to break. +# 1 million passwords per second against a modern box. This means that you +# should use very strong passwords, otherwise they will be very easy to break. +# Note that because the password is really a shared secret between the client +# and the server, and should not be memorized by any human, the password +# can be easily a long string from /dev/urandom or whatever, so by using a +# long and unguessable password no brute force attack will be possible. + +# Redis ACL users are defined in the following format: +# +# user ... acl rules ... +# +# For example: +# +# user worker +@list +@connection ~jobs:* on >ffa9203c493aa99 +# +# The special username "default" is used for new connections. If this user +# has the "nopass" rule, then new connections will be immediately authenticated +# as the "default" user without the need of any password provided via the +# AUTH command. Otherwise if the "default" user is not flagged with "nopass" +# the connections will start in not authenticated state, and will require +# AUTH (or the HELLO command AUTH option) in order to be authenticated and +# start to work. +# +# The ACL rules that describe what an user can do are the following: +# +# on Enable the user: it is possible to authenticate as this user. +# off Disable the user: it's no longer possible to authenticate +# with this user, however the already authenticated connections +# will still work. +# + Allow the execution of that command +# - Disallow the execution of that command +# +@ Allow the execution of all the commands in such category +# with valid categories are like @admin, @set, @sortedset, ... +# and so forth, see the full list in the server.c file where +# the Redis command table is described and defined. +# The special category @all means all the commands, but currently +# present in the server, and that will be loaded in the future +# via modules. +# +|subcommand Allow a specific subcommand of an otherwise +# disabled command. Note that this form is not +# allowed as negative like -DEBUG|SEGFAULT, but +# only additive starting with "+". +# allcommands Alias for +@all. Note that it implies the ability to execute +# all the future commands loaded via the modules system. +# nocommands Alias for -@all. +# ~ Add a pattern of keys that can be mentioned as part of +# commands. For instance ~* allows all the keys. The pattern +# is a glob-style pattern like the one of KEYS. +# It is possible to specify multiple patterns. +# allkeys Alias for ~* +# resetkeys Flush the list of allowed keys patterns. +# > Add this passowrd to the list of valid password for the user. +# For example >mypass will add "mypass" to the list. +# This directive clears the "nopass" flag (see later). +# < Remove this password from the list of valid passwords. +# nopass All the set passwords of the user are removed, and the user +# is flagged as requiring no password: it means that every +# password will work against this user. If this directive is +# used for the default user, every new connection will be +# immediately authenticated with the default user without +# any explicit AUTH command required. Note that the "resetpass" +# directive will clear this condition. +# resetpass Flush the list of allowed passwords. Moreover removes the +# "nopass" status. After "resetpass" the user has no associated +# passwords and there is no way to authenticate without adding +# some password (or setting it as "nopass" later). +# reset Performs the following actions: resetpass, resetkeys, off, +# -@all. The user returns to the same state it has immediately +# after its creation. +# +# ACL rules can be specified in any order: for instance you can start with +# passwords, then flags, or key patterns. However note that the additive +# and subtractive rules will CHANGE MEANING depending on the ordering. +# For instance see the following example: +# +# user alice on +@all -DEBUG ~* >somepassword +# +# This will allow "alice" to use all the commands with the exception of the +# DEBUG command, since +@all added all the commands to the set of the commands +# alice can use, and later DEBUG was removed. However if we invert the order +# of two ACL rules the result will be different: +# +# user alice on -DEBUG +@all ~* >somepassword +# +# Now DEBUG was removed when alice had yet no commands in the set of allowed +# commands, later all the commands are added, so the user will be able to +# execute everything. +# +# Basically ACL rules are processed left-to-right. +# +# For more information about ACL configuration please refer to +# the Redis web site at https://redis.io/topics/acl + +# ACL LOG +# +# The ACL Log tracks failed commands and authentication events associated +# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked +# by ACLs. The ACL Log is stored in and consumes memory. There is no limit +# to its length.You can reclaim memory with ACL LOG RESET or set a maximum +# length below. +acllog-max-len 128 + +# Using an external ACL file +# +# Instead of configuring users here in this file, it is possible to use +# a stand-alone file just listing users. The two methods cannot be mixed: +# if you configure users here and at the same time you activate the exteranl +# ACL file, the server will refuse to start. +# +# The format of the external ACL user file is exactly the same as the +# format that is used inside redis.conf to describe users. +# +# aclfile /etc/redis/users.acl + +# IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatiblity +# layer on top of the new ACL system. The option effect will be just setting +# the password for the default user. Clients will still authenticate using +# AUTH as usually, or more explicitly with AUTH default +# if they follow the new protocol: both will work. # # requirepass foobared -# Command renaming. +# Command renaming (DEPRECATED). +# +# ------------------------------------------------------------------------ +# WARNING: avoid using this option if possible. Instead use ACLs to remove +# commands from the default user, and put them only in some admin user you +# create for administrative purposes. +# ------------------------------------------------------------------------ # # It is possible to change the name of dangerous commands in a shared # environment. For instance the CONFIG command may be renamed into something @@ -566,13 +836,13 @@ replica-priority 100 # maxmemory # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory -# is reached. You can select among five behaviors: +# is reached. You can select one from the following behaviors: # -# volatile-lru -> Evict using approximated LRU among the keys with an expire set. +# volatile-lru -> Evict using approximated LRU, only keys with an expire set. # allkeys-lru -> Evict any key using approximated LRU. -# volatile-lfu -> Evict using approximated LFU among the keys with an expire set. +# volatile-lfu -> Evict using approximated LFU, only keys with an expire set. # allkeys-lfu -> Evict any key using approximated LFU. -# volatile-random -> Remove a random key among the ones with an expire set. +# volatile-random -> Remove a random key having an expire set. # allkeys-random -> Remove a random key, any key. # volatile-ttl -> Remove the key with the nearest expire time (minor TTL) # noeviction -> Don't evict anything, just return an error on write operations. @@ -613,20 +883,37 @@ replica-priority 100 # DEL commands to the replica as keys evict in the master side. # # This behavior ensures that masters and replicas stay consistent, and is usually -# what you want, however if your replica is writable, or you want the replica to have -# a different memory setting, and you are sure all the writes performed to the -# replica are idempotent, then you may change this default (but be sure to understand -# what you are doing). +# what you want, however if your replica is writable, or you want the replica +# to have a different memory setting, and you are sure all the writes performed +# to the replica are idempotent, then you may change this default (but be sure +# to understand what you are doing). # # Note that since the replica by default does not evict, it may end using more # memory than the one set via maxmemory (there are certain buffers that may -# be larger on the replica, or data structures may sometimes take more memory and so -# forth). So make sure you monitor your replicas and make sure they have enough -# memory to never hit a real out-of-memory condition before the master hits -# the configured maxmemory setting. +# be larger on the replica, or data structures may sometimes take more memory +# and so forth). So make sure you monitor your replicas and make sure they +# have enough memory to never hit a real out-of-memory condition before the +# master hits the configured maxmemory setting. # # replica-ignore-maxmemory yes +# Redis reclaims expired keys in two ways: upon access when those keys are +# found to be expired, and also in background, in what is called the +# "active expire key". The key space is slowly and interactively scanned +# looking for expired keys to reclaim, so that it is possible to free memory +# of keys that are expired and will never be accessed again in a short time. +# +# The default effort of the expire cycle will try to avoid having more than +# ten percent of expired keys still in memory, and will try to avoid consuming +# more than 25% of total memory and to add latency to the system. However +# it is possible to increase the expire "effort" that is normally set to +# "1", to a greater value, up to the value "10". At its maximum value the +# system will use more CPU, longer cycles (and technically may introduce +# more latency), and will tollerate less already expired keys still present +# in the system. It's a tradeoff betweeen memory, CPU and latecy. +# +# active-expire-effort 1 + ############################# LAZY FREEING #################################### # Redis has two primitives to delete keys. One is called DEL and is a blocking @@ -669,13 +956,66 @@ replica-priority 100 # In all the above cases the default is to delete objects in a blocking way, # like if DEL was called. However you can configure each case specifically # in order to instead release memory in a non-blocking way like if UNLINK -# was called, using the following configuration directives: +# was called, using the following configuration directives. lazyfree-lazy-eviction no lazyfree-lazy-expire no lazyfree-lazy-server-del no replica-lazy-flush no +# It is also possible, for the case when to replace the user code DEL calls +# with UNLINK calls is not easy, to modify the default behavior of the DEL +# command to act exactly like UNLINK, using the following configuration +# directive: + +lazyfree-lazy-user-del no + +################################ THREADED I/O ################################# + +# Redis is mostly single threaded, however there are certain threaded +# operations such as UNLINK, slow I/O accesses and other things that are +# performed on side threads. +# +# Now it is also possible to handle Redis clients socket reads and writes +# in different I/O threads. Since especially writing is so slow, normally +# Redis users use pipelining in order to speedup the Redis performances per +# core, and spawn multiple instances in order to scale more. Using I/O +# threads it is possible to easily speedup two times Redis without resorting +# to pipelining nor sharding of the instance. +# +# By default threading is disabled, we suggest enabling it only in machines +# that have at least 4 or more cores, leaving at least one spare core. +# Using more than 8 threads is unlikely to help much. We also recommend using +# threaded I/O only if you actually have performance problems, with Redis +# instances being able to use a quite big percentage of CPU time, otherwise +# there is no point in using this feature. +# +# So for instance if you have a four cores boxes, try to use 2 or 3 I/O +# threads, if you have a 8 cores, try to use 6 threads. In order to +# enable I/O threads use the following configuration directive: +# +# io-threads 4 +# +# Setting io-threads to 1 will just use the main thread as usually. +# When I/O threads are enabled, we only use threads for writes, that is +# to thread the write(2) syscall and transfer the client buffers to the +# socket. However it is also possible to enable threading of reads and +# protocol parsing using the following configuration directive, by setting +# it to yes: +# +# io-threads-do-reads no +# +# Usually threading reads doesn't help much. +# +# NOTE 1: This configuration directive cannot be changed at runtime via +# CONFIG SET. Aso this feature currently does not work when SSL is +# enabled. +# +# NOTE 2: If you want to test the Redis speedup using redis-benchmark, make +# sure you also run the benchmark itself in threaded mode, using the +# --threads option to match the number of Redis theads, otherwise you'll not +# be able to notice the improvements. + ############################## APPEND ONLY MODE ############################### # By default Redis asynchronously dumps the dataset on disk. This mode is @@ -932,6 +1272,22 @@ lua-time-limit 5000 # # cluster-replica-no-failover no +# This option, when set to yes, allows nodes to serve read traffic while the +# the cluster is in a down state, as long as it believes it owns the slots. +# +# This is useful for two cases. The first case is for when an application +# doesn't require consistency of data during node failures or network partitions. +# One example of this is a cache, where as long as the node has the data it +# should be able to serve it. +# +# The second use case is for configurations that don't meet the recommended +# three shards but want to enable cluster mode and scale later. A +# master outage in a 1 or 2 shard configuration causes a read/write outage to the +# entire cluster without this option set, with it set there is only a write outage. +# Without a quorum of masters, slot ownership will not change automatically. +# +# cluster-allow-reads-when-down no + # In order to setup your cluster make sure to read the documentation # available at http://redis.io web site. @@ -1038,7 +1394,11 @@ latency-monitor-threshold 0 # z Sorted set commands # x Expired events (events generated every time a key expires) # e Evicted events (events generated when a key is evicted for maxmemory) -# A Alias for g$lshzxe, so that the "AKE" string means all the events. +# t Stream commands +# m Key-miss events (Note: It is not included in the 'A' class) +# A Alias for g$lshzxet, so that the "AKE" string means all the events +# (Except key-miss events which are excluded from 'A' due to their +# unique nature). # # The "notify-keyspace-events" takes as argument a string that is composed # of zero or multiple characters. The empty string means that notifications @@ -1059,6 +1419,61 @@ latency-monitor-threshold 0 # specify at least one of K or E, no events will be delivered. notify-keyspace-events "" +############################### GOPHER SERVER ################################# + +# Redis contains an implementation of the Gopher protocol, as specified in +# the RFC 1436 (https://www.ietf.org/rfc/rfc1436.txt). +# +# The Gopher protocol was very popular in the late '90s. It is an alternative +# to the web, and the implementation both server and client side is so simple +# that the Redis server has just 100 lines of code in order to implement this +# support. +# +# What do you do with Gopher nowadays? Well Gopher never *really* died, and +# lately there is a movement in order for the Gopher more hierarchical content +# composed of just plain text documents to be resurrected. Some want a simpler +# internet, others believe that the mainstream internet became too much +# controlled, and it's cool to create an alternative space for people that +# want a bit of fresh air. +# +# Anyway for the 10nth birthday of the Redis, we gave it the Gopher protocol +# as a gift. +# +# --- HOW IT WORKS? --- +# +# The Redis Gopher support uses the inline protocol of Redis, and specifically +# two kind of inline requests that were anyway illegal: an empty request +# or any request that starts with "/" (there are no Redis commands starting +# with such a slash). Normal RESP2/RESP3 requests are completely out of the +# path of the Gopher protocol implementation and are served as usually as well. +# +# If you open a connection to Redis when Gopher is enabled and send it +# a string like "/foo", if there is a key named "/foo" it is served via the +# Gopher protocol. +# +# In order to create a real Gopher "hole" (the name of a Gopher site in Gopher +# talking), you likely need a script like the following: +# +# https://github.com/antirez/gopher2redis +# +# --- SECURITY WARNING --- +# +# If you plan to put Redis on the internet in a publicly accessible address +# to server Gopher pages MAKE SURE TO SET A PASSWORD to the instance. +# Once a password is set: +# +# 1. The Gopher server (when enabled, not by default) will still serve +# content via Gopher. +# 2. However other commands cannot be called before the client will +# authenticate. +# +# So use the 'requirepass' option to protect your instance. +# +# To enable Gopher support uncomment the following line and set +# the option from no (the default) to yes. +# +# gopher-enabled no + ############################### ADVANCED CONFIG ############################### # Hashes are encoded using a memory efficient data structure when they have a @@ -1233,7 +1648,7 @@ hz 10 # offers, and enables by default, the ability to use an adaptive HZ value # which will temporary raise when there are many connected clients. # -# When dynamic HZ is enabled, the actual configured HZ will be used as +# When dynamic HZ is enabled, the actual configured HZ will be used # as a baseline, but multiples of the configured HZ value will be actually # used as needed once more clients are connected. In this way an idle # instance will use very little CPU time while a busy instance will be @@ -1306,10 +1721,6 @@ rdb-save-incremental-fsync yes ########################### ACTIVE DEFRAGMENTATION ####################### # -# WARNING THIS FEATURE IS EXPERIMENTAL. However it was stress tested -# even in production and manually tested by multiple engineers for some -# time. -# # What is active defragmentation? # ------------------------------- # @@ -1349,7 +1760,7 @@ rdb-save-incremental-fsync yes # a good idea to leave the defaults untouched. # Enabled active defragmentation -# activedefrag yes +# activedefrag no # Minimum amount of fragmentation waste to start active defrag # active-defrag-ignore-bytes 100mb @@ -1360,13 +1771,42 @@ rdb-save-incremental-fsync yes # Maximum percentage of fragmentation at which we use maximum effort # active-defrag-threshold-upper 100 -# Minimal effort for defrag in CPU percentage -# active-defrag-cycle-min 5 +# Minimal effort for defrag in CPU percentage, to be used when the lower +# threshold is reached +# active-defrag-cycle-min 1 -# Maximal effort for defrag in CPU percentage -# active-defrag-cycle-max 75 +# Maximal effort for defrag in CPU percentage, to be used when the upper +# threshold is reached +# active-defrag-cycle-max 25 # Maximum number of set/hash/zset/list fields that will be processed from # the main dictionary scan # active-defrag-max-scan-fields 1000 +# Jemalloc background thread for purging will be enabled by default +jemalloc-bg-thread yes + +# It is possible to pin different threads and processes of Redis to specific +# CPUs in your system, in order to maximize the performances of the server. +# This is useful both in order to pin different Redis threads in different +# CPUs, but also in order to make sure that multiple Redis instances running +# in the same host will be pinned to different CPUs. +# +# Normally you can do this using the "taskset" command, however it is also +# possible to this via Redis configuration directly, both in Linux and FreeBSD. +# +# You can pin the server/IO threads, bio threads, aof rewrite child process, and +# the bgsave child process. The syntax to specify the cpu list is the same as +# the taskset command: +# +# Set redis server/io threads to cpu affinity 0,2,4,6: +# server_cpulist 0-7:2 +# +# Set bio threads to cpu affinity 1,3: +# bio_cpulist 1,3 +# +# Set aof rewrite child process to cpu affinity 8,9,10,11: +# aof_rewrite_cpulist 8-11 +# +# Set bgsave child process to cpu affinity 1,10,11 +# bgsave_cpulist 1,10-11 diff --git a/redis.submodule/runtest-moduleapi b/redis.submodule/runtest-moduleapi index 8e1c0cb..f6cc0a2 100755 --- a/redis.submodule/runtest-moduleapi +++ b/redis.submodule/runtest-moduleapi @@ -13,4 +13,16 @@ then fi make -C tests/modules && \ -$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/testrdb "${@}" +$TCLSH tests/test_helper.tcl \ +--single unit/moduleapi/commandfilter \ +--single unit/moduleapi/fork \ +--single unit/moduleapi/testrdb \ +--single unit/moduleapi/infotest \ +--single unit/moduleapi/propagate \ +--single unit/moduleapi/hooks \ +--single unit/moduleapi/misc \ +--single unit/moduleapi/blockonkeys \ +--single unit/moduleapi/scan \ +--single unit/moduleapi/datatype \ +--single unit/moduleapi/auth \ +"${@}" diff --git a/redis.submodule/sentinel.conf b/redis.submodule/sentinel.conf index bc9a705..4ca5e5f 100644 --- a/redis.submodule/sentinel.conf +++ b/redis.submodule/sentinel.conf @@ -102,6 +102,18 @@ sentinel monitor mymaster 127.0.0.1 6379 2 # # sentinel auth-pass mymaster MySUPER--secret-0123passw0rd +# sentinel auth-user +# +# This is useful in order to authenticate to instances having ACL capabilities, +# that is, running Redis 6.0 or greater. When just auth-pass is provided the +# Sentinel instance will authenticate to Redis using the old "AUTH " +# method. When also an username is provided, it will use "AUTH ". +# In the Redis servers side, the ACL to provide just minimal access to +# Sentinel instances, should be configured along the following lines: +# +# user sentinel-user >somepassword +client +subscribe +publish \ +# +ping +info +multi +slaveof +config +client +exec on + # sentinel down-after-milliseconds # # Number of milliseconds the master (or any attached replica or sentinel) should @@ -112,6 +124,14 @@ sentinel monitor mymaster 127.0.0.1 6379 2 # Default is 30 seconds. sentinel down-after-milliseconds mymaster 30000 +# requirepass +# +# You can configure Sentinel itself to require a password, however when doing +# so Sentinel will try to authenticate with the same password to all the +# other Sentinels. So you need to configure all your Sentinels in a given +# group with the same "requirepass" password. Check the following documentation +# for more info: https://redis.io/topics/sentinel + # sentinel parallel-syncs # # How many replicas we can reconfigure to point to the new replica simultaneously diff --git a/redis.submodule/src/Makefile b/redis.submodule/src/Makefile index 2a68649..b8c05c3 100644 --- a/redis.submodule/src/Makefile +++ b/redis.submodule/src/Makefile @@ -20,7 +20,7 @@ DEPENDENCY_TARGETS=hiredis linenoise lua NODEPS:=clean distclean # Default settings -STD=-std=c99 -pedantic -DREDIS_STATIC='' +STD=-std=c11 -pedantic -DREDIS_STATIC='' ifneq (,$(findstring clang,$(CC))) ifneq (,$(findstring FreeBSD,$(uname_S))) STD+=-Wno-c11-extensions @@ -32,6 +32,7 @@ OPT=$(OPTIMIZATION) PREFIX?=/usr/local INSTALL_BIN=$(PREFIX)/bin INSTALL=install +PKG_CONFIG?=pkg-config # Default allocator defaults to Jemalloc if it's not an ARM MALLOC=libc @@ -77,6 +78,15 @@ FINAL_LDFLAGS=$(LDFLAGS) $(REDIS_LDFLAGS) $(DEBUG) FINAL_LIBS=-lm DEBUG=-g -ggdb +# Linux ARM needs -latomic at linking time +ifneq (,$(filter aarch64 armv,$(uname_M))) + FINAL_LIBS+=-latomic +else +ifneq (,$(findstring armv,$(uname_M))) + FINAL_LIBS+=-latomic +endif +endif + ifeq ($(uname_S),SunOS) # SunOS ifneq ($(@@),32bit) @@ -93,6 +103,8 @@ else ifeq ($(uname_S),Darwin) # Darwin FINAL_LIBS+= -ldl + OPENSSL_CFLAGS=-I/usr/local/opt/openssl/include + OPENSSL_LDFLAGS=-L/usr/local/opt/openssl/lib else ifeq ($(uname_S),AIX) # AIX @@ -116,6 +128,14 @@ else ifeq ($(uname_S),DragonFly) # FreeBSD FINAL_LIBS+= -lpthread -lexecinfo +else +ifeq ($(uname_S),OpenBSD) + # OpenBSD + FINAL_LIBS+= -lpthread -lexecinfo +else +ifeq ($(uname_S),NetBSD) + # NetBSD + FINAL_LIBS+= -lpthread -lexecinfo else # All the other OSes (notably Linux) FINAL_LDFLAGS+= -rdynamic @@ -126,9 +146,35 @@ endif endif endif endif +endif +endif # Include paths to dependencies FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src +# Determine systemd support and/or build preference (defaulting to auto-detection) +BUILD_WITH_SYSTEMD=no +# If 'USE_SYSTEMD' in the environment is neither "no" nor "yes", try to +# auto-detect libsystemd's presence and link accordingly. +ifneq ($(USE_SYSTEMD),no) + LIBSYSTEMD_PKGCONFIG := $(shell $(PKG_CONFIG) --exists libsystemd && echo $$?) +# If libsystemd cannot be detected, continue building without support for it +# (unless a later check tells us otherwise) +ifeq ($(LIBSYSTEMD_PKGCONFIG),0) + BUILD_WITH_SYSTEMD=yes +endif +endif +ifeq ($(USE_SYSTEMD),yes) +ifneq ($(LIBSYSTEMD_PKGCONFIG),0) +$(error USE_SYSTEMD is set to "$(USE_SYSTEMD)", but $(PKG_CONFIG) cannot find libsystemd) +endif +# Force building with libsystemd + BUILD_WITH_SYSTEMD=yes +endif +ifeq ($(BUILD_WITH_SYSTEMD),yes) + FINAL_LIBS+=$(shell $(PKG_CONFIG) --libs libsystemd) + FINAL_CFLAGS+= -DHAVE_LIBSYSTEMD +endif + ifeq ($(MALLOC),tcmalloc) FINAL_CFLAGS+= -DUSE_TCMALLOC FINAL_LIBS+= -ltcmalloc @@ -145,6 +191,12 @@ ifeq ($(MALLOC),jemalloc) FINAL_LIBS := ../deps/jemalloc/lib/libjemalloc.a $(FINAL_LIBS) endif +ifeq ($(BUILD_TLS),yes) + FINAL_CFLAGS+=-DUSE_OPENSSL $(OPENSSL_CFLAGS) + FINAL_LDFLAGS+=$(OPENSSL_LDFLAGS) + FINAL_LIBS += ../deps/hiredis/libhiredis_ssl.a -lssl -lcrypto +endif + REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS) REDIS_INSTALL=$(QUIET_INSTALL)$(INSTALL) @@ -164,11 +216,11 @@ endif REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o REDIS_CLI_NAME=redis-cli -REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o +REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o REDIS_BENCHMARK_NAME=redis-benchmark -REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o zmalloc.o redis-benchmark.o +REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o REDIS_CHECK_RDB_NAME=redis-check-rdb REDIS_CHECK_AOF_NAME=redis-check-aof @@ -241,14 +293,18 @@ $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ) dict-benchmark: dict.c zmalloc.c sds.c siphash.c $(REDIS_CC) $(FINAL_CFLAGS) $^ -D DICT_BENCHMARK_MAIN -o $@ $(FINAL_LIBS) +DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d) +-include $(DEP) + # Because the jemalloc.h header is generated as a part of the jemalloc build, # building it should complete before building any other object. Instead of # depending on a single artifact, build all dependencies first. %.o: %.c .make-prerequisites - $(REDIS_CC) -c $< + $(REDIS_CC) -MMD -o $@ -c $< clean: rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html Makefile.dep dict-benchmark + rm -f $(DEP) .PHONY: clean diff --git a/redis.submodule/src/acl.c b/redis.submodule/src/acl.c new file mode 100644 index 0000000..6dd0f70 --- /dev/null +++ b/redis.submodule/src/acl.c @@ -0,0 +1,1990 @@ +/* + * Copyright (c) 2018, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "server.h" +#include "sha256.h" +#include +#include + +/* ============================================================================= + * Global state for ACLs + * ==========================================================================*/ + +rax *Users; /* Table mapping usernames to user structures. */ + +user *DefaultUser; /* Global reference to the default user. + Every new connection is associated to it, if no + AUTH or HELLO is used to authenticate with a + different user. */ + +list *UsersToLoad; /* This is a list of users found in the configuration file + that we'll need to load in the final stage of Redis + initialization, after all the modules are already + loaded. Every list element is a NULL terminated + array of SDS pointers: the first is the user name, + all the remaining pointers are ACL rules in the same + format as ACLSetUser(). */ +list *ACLLog; /* Our security log, the user is able to inspect that + using the ACL LOG command .*/ + +struct ACLCategoryItem { + const char *name; + uint64_t flag; +} ACLCommandCategories[] = { + {"keyspace", CMD_CATEGORY_KEYSPACE}, + {"read", CMD_CATEGORY_READ}, + {"write", CMD_CATEGORY_WRITE}, + {"set", CMD_CATEGORY_SET}, + {"sortedset", CMD_CATEGORY_SORTEDSET}, + {"list", CMD_CATEGORY_LIST}, + {"hash", CMD_CATEGORY_HASH}, + {"string", CMD_CATEGORY_STRING}, + {"bitmap", CMD_CATEGORY_BITMAP}, + {"hyperloglog", CMD_CATEGORY_HYPERLOGLOG}, + {"geo", CMD_CATEGORY_GEO}, + {"stream", CMD_CATEGORY_STREAM}, + {"pubsub", CMD_CATEGORY_PUBSUB}, + {"admin", CMD_CATEGORY_ADMIN}, + {"fast", CMD_CATEGORY_FAST}, + {"slow", CMD_CATEGORY_SLOW}, + {"blocking", CMD_CATEGORY_BLOCKING}, + {"dangerous", CMD_CATEGORY_DANGEROUS}, + {"connection", CMD_CATEGORY_CONNECTION}, + {"transaction", CMD_CATEGORY_TRANSACTION}, + {"scripting", CMD_CATEGORY_SCRIPTING}, + {NULL,0} /* Terminator. */ +}; + +struct ACLUserFlag { + const char *name; + uint64_t flag; +} ACLUserFlags[] = { + {"on", USER_FLAG_ENABLED}, + {"off", USER_FLAG_DISABLED}, + {"allkeys", USER_FLAG_ALLKEYS}, + {"allcommands", USER_FLAG_ALLCOMMANDS}, + {"nopass", USER_FLAG_NOPASS}, + {NULL,0} /* Terminator. */ +}; + +void ACLResetSubcommandsForCommand(user *u, unsigned long id); +void ACLResetSubcommands(user *u); +void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub); +void ACLFreeLogEntry(void *le); + +/* The length of the string representation of a hashed password. */ +#define HASH_PASSWORD_LEN SHA256_BLOCK_SIZE*2 + +/* ============================================================================= + * Helper functions for the rest of the ACL implementation + * ==========================================================================*/ + +/* Return zero if strings are the same, non-zero if they are not. + * The comparison is performed in a way that prevents an attacker to obtain + * information about the nature of the strings just monitoring the execution + * time of the function. + * + * Note that limiting the comparison length to strings up to 512 bytes we + * can avoid leaking any information about the password length and any + * possible branch misprediction related leak. + */ +int time_independent_strcmp(char *a, char *b) { + char bufa[CONFIG_AUTHPASS_MAX_LEN], bufb[CONFIG_AUTHPASS_MAX_LEN]; + /* The above two strlen perform len(a) + len(b) operations where either + * a or b are fixed (our password) length, and the difference is only + * relative to the length of the user provided string, so no information + * leak is possible in the following two lines of code. */ + unsigned int alen = strlen(a); + unsigned int blen = strlen(b); + unsigned int j; + int diff = 0; + + /* We can't compare strings longer than our static buffers. + * Note that this will never pass the first test in practical circumstances + * so there is no info leak. */ + if (alen > sizeof(bufa) || blen > sizeof(bufb)) return 1; + + memset(bufa,0,sizeof(bufa)); /* Constant time. */ + memset(bufb,0,sizeof(bufb)); /* Constant time. */ + /* Again the time of the following two copies is proportional to + * len(a) + len(b) so no info is leaked. */ + memcpy(bufa,a,alen); + memcpy(bufb,b,blen); + + /* Always compare all the chars in the two buffers without + * conditional expressions. */ + for (j = 0; j < sizeof(bufa); j++) { + diff |= (bufa[j] ^ bufb[j]); + } + /* Length must be equal as well. */ + diff |= alen ^ blen; + return diff; /* If zero strings are the same. */ +} + +/* Given an SDS string, returns the SHA256 hex representation as a + * new SDS string. */ +sds ACLHashPassword(unsigned char *cleartext, size_t len) { + SHA256_CTX ctx; + unsigned char hash[SHA256_BLOCK_SIZE]; + char hex[HASH_PASSWORD_LEN]; + char *cset = "0123456789abcdef"; + + sha256_init(&ctx); + sha256_update(&ctx,(unsigned char*)cleartext,len); + sha256_final(&ctx,hash); + + for (int j = 0; j < SHA256_BLOCK_SIZE; j++) { + hex[j*2] = cset[((hash[j]&0xF0)>>4)]; + hex[j*2+1] = cset[(hash[j]&0xF)]; + } + return sdsnewlen(hex,HASH_PASSWORD_LEN); +} + +/* Given a hash and the hash length, returns C_OK if it is a valid password + * hash, or C_ERR otherwise. */ +int ACLCheckPasswordHash(unsigned char *hash, int hashlen) { + if (hashlen != HASH_PASSWORD_LEN) { + return C_ERR; + } + + /* Password hashes can only be characters that represent + * hexadecimal values, which are numbers and lowercase + * characters 'a' through 'f'. */ + for(int i = 0; i < HASH_PASSWORD_LEN; i++) { + char c = hash[i]; + if ((c < 'a' || c > 'f') && (c < '0' || c > '9')) { + return C_ERR; + } + } + return C_OK; +} + +/* ============================================================================= + * Low level ACL API + * ==========================================================================*/ + +/* Return 1 if the specified string contains spaces or null characters. + * We do this for usernames and key patterns for simpler rewriting of + * ACL rules, presentation on ACL list, and to avoid subtle security bugs + * that may arise from parsing the rules in presence of escapes. + * The function returns 0 if the string has no spaces. */ +int ACLStringHasSpaces(const char *s, size_t len) { + for (size_t i = 0; i < len; i++) { + if (isspace(s[i]) || s[i] == 0) return 1; + } + return 0; +} + +/* Given the category name the command returns the corresponding flag, or + * zero if there is no match. */ +uint64_t ACLGetCommandCategoryFlagByName(const char *name) { + for (int j = 0; ACLCommandCategories[j].flag != 0; j++) { + if (!strcasecmp(name,ACLCommandCategories[j].name)) { + return ACLCommandCategories[j].flag; + } + } + return 0; /* No match. */ +} + +/* Method for passwords/pattern comparison used for the user->passwords list + * so that we can search for items with listSearchKey(). */ +int ACLListMatchSds(void *a, void *b) { + return sdscmp(a,b) == 0; +} + +/* Method to free list elements from ACL users password/patterns lists. */ +void ACLListFreeSds(void *item) { + sdsfree(item); +} + +/* Method to duplicate list elements from ACL users password/patterns lists. */ +void *ACLListDupSds(void *item) { + return sdsdup(item); +} + +/* Create a new user with the specified name, store it in the list + * of users (the Users global radix tree), and returns a reference to + * the structure representing the user. + * + * If the user with such name already exists NULL is returned. */ +user *ACLCreateUser(const char *name, size_t namelen) { + if (raxFind(Users,(unsigned char*)name,namelen) != raxNotFound) return NULL; + user *u = zmalloc(sizeof(*u)); + u->name = sdsnewlen(name,namelen); + u->flags = USER_FLAG_DISABLED; + u->allowed_subcommands = NULL; + u->passwords = listCreate(); + u->patterns = listCreate(); + listSetMatchMethod(u->passwords,ACLListMatchSds); + listSetFreeMethod(u->passwords,ACLListFreeSds); + listSetDupMethod(u->passwords,ACLListDupSds); + listSetMatchMethod(u->patterns,ACLListMatchSds); + listSetFreeMethod(u->patterns,ACLListFreeSds); + listSetDupMethod(u->patterns,ACLListDupSds); + memset(u->allowed_commands,0,sizeof(u->allowed_commands)); + raxInsert(Users,(unsigned char*)name,namelen,u,NULL); + return u; +} + +/* This function should be called when we need an unlinked "fake" user + * we can use in order to validate ACL rules or for other similar reasons. + * The user will not get linked to the Users radix tree. The returned + * user should be released with ACLFreeUser() as usually. */ +user *ACLCreateUnlinkedUser(void) { + char username[64]; + for (int j = 0; ; j++) { + snprintf(username,sizeof(username),"__fakeuser:%d__",j); + user *fakeuser = ACLCreateUser(username,strlen(username)); + if (fakeuser == NULL) continue; + int retval = raxRemove(Users,(unsigned char*) username, + strlen(username),NULL); + serverAssert(retval != 0); + return fakeuser; + } +} + +/* Release the memory used by the user structure. Note that this function + * will not remove the user from the Users global radix tree. */ +void ACLFreeUser(user *u) { + sdsfree(u->name); + listRelease(u->passwords); + listRelease(u->patterns); + ACLResetSubcommands(u); + zfree(u); +} + +/* When a user is deleted we need to cycle the active + * connections in order to kill all the pending ones that + * are authenticated with such user. */ +void ACLFreeUserAndKillClients(user *u) { + listIter li; + listNode *ln; + listRewind(server.clients,&li); + while ((ln = listNext(&li)) != NULL) { + client *c = listNodeValue(ln); + if (c->user == u) { + /* We'll free the conenction asynchronously, so + * in theory to set a different user is not needed. + * However if there are bugs in Redis, soon or later + * this may result in some security hole: it's much + * more defensive to set the default user and put + * it in non authenticated mode. */ + c->user = DefaultUser; + c->authenticated = 0; + freeClientAsync(c); + } + } + ACLFreeUser(u); +} + +/* Copy the user ACL rules from the source user 'src' to the destination + * user 'dst' so that at the end of the process they'll have exactly the + * same rules (but the names will continue to be the original ones). */ +void ACLCopyUser(user *dst, user *src) { + listRelease(dst->passwords); + listRelease(dst->patterns); + dst->passwords = listDup(src->passwords); + dst->patterns = listDup(src->patterns); + memcpy(dst->allowed_commands,src->allowed_commands, + sizeof(dst->allowed_commands)); + dst->flags = src->flags; + ACLResetSubcommands(dst); + /* Copy the allowed subcommands array of array of SDS strings. */ + if (src->allowed_subcommands) { + for (int j = 0; j < USER_COMMAND_BITS_COUNT; j++) { + if (src->allowed_subcommands[j]) { + for (int i = 0; src->allowed_subcommands[j][i]; i++) + { + ACLAddAllowedSubcommand(dst, j, + src->allowed_subcommands[j][i]); + } + } + } + } +} + +/* Free all the users registered in the radix tree 'users' and free the + * radix tree itself. */ +void ACLFreeUsersSet(rax *users) { + raxFreeWithCallback(users,(void(*)(void*))ACLFreeUserAndKillClients); +} + +/* Given a command ID, this function set by reference 'word' and 'bit' + * so that user->allowed_commands[word] will address the right word + * where the corresponding bit for the provided ID is stored, and + * so that user->allowed_commands[word]&bit will identify that specific + * bit. The function returns C_ERR in case the specified ID overflows + * the bitmap in the user representation. */ +int ACLGetCommandBitCoordinates(uint64_t id, uint64_t *word, uint64_t *bit) { + if (id >= USER_COMMAND_BITS_COUNT) return C_ERR; + *word = id / sizeof(uint64_t) / 8; + *bit = 1ULL << (id % (sizeof(uint64_t) * 8)); + return C_OK; +} + +/* Check if the specified command bit is set for the specified user. + * The function returns 1 is the bit is set or 0 if it is not. + * Note that this function does not check the ALLCOMMANDS flag of the user + * but just the lowlevel bitmask. + * + * If the bit overflows the user internal representation, zero is returned + * in order to disallow the execution of the command in such edge case. */ +int ACLGetUserCommandBit(user *u, unsigned long id) { + uint64_t word, bit; + if (ACLGetCommandBitCoordinates(id,&word,&bit) == C_ERR) return 0; + return (u->allowed_commands[word] & bit) != 0; +} + +/* When +@all or allcommands is given, we set a reserved bit as well that we + * can later test, to see if the user has the right to execute "future commands", + * that is, commands loaded later via modules. */ +int ACLUserCanExecuteFutureCommands(user *u) { + return ACLGetUserCommandBit(u,USER_COMMAND_BITS_COUNT-1); +} + +/* Set the specified command bit for the specified user to 'value' (0 or 1). + * If the bit overflows the user internal representation, no operation + * is performed. As a side effect of calling this function with a value of + * zero, the user flag ALLCOMMANDS is cleared since it is no longer possible + * to skip the command bit explicit test. */ +void ACLSetUserCommandBit(user *u, unsigned long id, int value) { + uint64_t word, bit; + if (ACLGetCommandBitCoordinates(id,&word,&bit) == C_ERR) return; + if (value) { + u->allowed_commands[word] |= bit; + } else { + u->allowed_commands[word] &= ~bit; + u->flags &= ~USER_FLAG_ALLCOMMANDS; + } +} + +/* This is like ACLSetUserCommandBit(), but instead of setting the specified + * ID, it will check all the commands in the category specified as argument, + * and will set all the bits corresponding to such commands to the specified + * value. Since the category passed by the user may be non existing, the + * function returns C_ERR if the category was not found, or C_OK if it was + * found and the operation was performed. */ +int ACLSetUserCommandBitsForCategory(user *u, const char *category, int value) { + uint64_t cflag = ACLGetCommandCategoryFlagByName(category); + if (!cflag) return C_ERR; + dictIterator *di = dictGetIterator(server.orig_commands); + dictEntry *de; + while ((de = dictNext(di)) != NULL) { + struct redisCommand *cmd = dictGetVal(de); + if (cmd->flags & CMD_MODULE) continue; /* Ignore modules commands. */ + if (cmd->flags & cflag) { + ACLSetUserCommandBit(u,cmd->id,value); + ACLResetSubcommandsForCommand(u,cmd->id); + } + } + dictReleaseIterator(di); + return C_OK; +} + +/* Return the number of commands allowed (on) and denied (off) for the user 'u' + * in the subset of commands flagged with the specified category name. + * If the category name is not valid, C_ERR is returned, otherwise C_OK is + * returned and on and off are populated by reference. */ +int ACLCountCategoryBitsForUser(user *u, unsigned long *on, unsigned long *off, + const char *category) +{ + uint64_t cflag = ACLGetCommandCategoryFlagByName(category); + if (!cflag) return C_ERR; + + *on = *off = 0; + dictIterator *di = dictGetIterator(server.orig_commands); + dictEntry *de; + while ((de = dictNext(di)) != NULL) { + struct redisCommand *cmd = dictGetVal(de); + if (cmd->flags & cflag) { + if (ACLGetUserCommandBit(u,cmd->id)) + (*on)++; + else + (*off)++; + } + } + dictReleaseIterator(di); + return C_OK; +} + +/* This function returns an SDS string representing the specified user ACL + * rules related to command execution, in the same format you could set them + * back using ACL SETUSER. The function will return just the set of rules needed + * to recreate the user commands bitmap, without including other user flags such + * as on/off, passwords and so forth. The returned string always starts with + * the +@all or -@all rule, depending on the user bitmap, and is followed, if + * needed, by the other rules needed to narrow or extend what the user can do. */ +sds ACLDescribeUserCommandRules(user *u) { + sds rules = sdsempty(); + int additive; /* If true we start from -@all and add, otherwise if + false we start from +@all and remove. */ + + /* This code is based on a trick: as we generate the rules, we apply + * them to a fake user, so that as we go we still know what are the + * bit differences we should try to address by emitting more rules. */ + user fu = {0}; + user *fakeuser = &fu; + + /* Here we want to understand if we should start with +@all and remove + * the commands corresponding to the bits that are not set in the user + * commands bitmap, or the contrary. Note that semantically the two are + * different. For instance starting with +@all and subtracting, the user + * will be able to execute future commands, while -@all and adding will just + * allow the user the run the selected commands and/or categories. + * How do we test for that? We use the trick of a reserved command ID bit + * that is set only by +@all (and its alias "allcommands"). */ + if (ACLUserCanExecuteFutureCommands(u)) { + additive = 0; + rules = sdscat(rules,"+@all "); + ACLSetUser(fakeuser,"+@all",-1); + } else { + additive = 1; + rules = sdscat(rules,"-@all "); + ACLSetUser(fakeuser,"-@all",-1); + } + + /* Try to add or subtract each category one after the other. Often a + * single category will not perfectly match the set of commands into + * it, so at the end we do a final pass adding/removing the single commands + * needed to make the bitmap exactly match. */ + for (int j = 0; ACLCommandCategories[j].flag != 0; j++) { + unsigned long on, off; + ACLCountCategoryBitsForUser(u,&on,&off,ACLCommandCategories[j].name); + if ((additive && on > off) || (!additive && off > on)) { + sds op = sdsnewlen(additive ? "+@" : "-@", 2); + op = sdscat(op,ACLCommandCategories[j].name); + ACLSetUser(fakeuser,op,-1); + rules = sdscatsds(rules,op); + rules = sdscatlen(rules," ",1); + sdsfree(op); + } + } + + /* Fix the final ACLs with single commands differences. */ + dictIterator *di = dictGetIterator(server.orig_commands); + dictEntry *de; + while ((de = dictNext(di)) != NULL) { + struct redisCommand *cmd = dictGetVal(de); + int userbit = ACLGetUserCommandBit(u,cmd->id); + int fakebit = ACLGetUserCommandBit(fakeuser,cmd->id); + if (userbit != fakebit) { + rules = sdscatlen(rules, userbit ? "+" : "-", 1); + rules = sdscat(rules,cmd->name); + rules = sdscatlen(rules," ",1); + ACLSetUserCommandBit(fakeuser,cmd->id,userbit); + } + + /* Emit the subcommands if there are any. */ + if (userbit == 0 && u->allowed_subcommands && + u->allowed_subcommands[cmd->id]) + { + for (int j = 0; u->allowed_subcommands[cmd->id][j]; j++) { + rules = sdscatlen(rules,"+",1); + rules = sdscat(rules,cmd->name); + rules = sdscatlen(rules,"|",1); + rules = sdscatsds(rules,u->allowed_subcommands[cmd->id][j]); + rules = sdscatlen(rules," ",1); + } + } + } + dictReleaseIterator(di); + + /* Trim the final useless space. */ + sdsrange(rules,0,-2); + + /* This is technically not needed, but we want to verify that now the + * predicted bitmap is exactly the same as the user bitmap, and abort + * otherwise, because aborting is better than a security risk in this + * code path. */ + if (memcmp(fakeuser->allowed_commands, + u->allowed_commands, + sizeof(u->allowed_commands)) != 0) + { + serverLog(LL_WARNING, + "CRITICAL ERROR: User ACLs don't match final bitmap: '%s'", + rules); + serverPanic("No bitmap match in ACLDescribeUserCommandRules()"); + } + return rules; +} + +/* This is similar to ACLDescribeUserCommandRules(), however instead of + * describing just the user command rules, everything is described: user + * flags, keys, passwords and finally the command rules obtained via + * the ACLDescribeUserCommandRules() function. This is the function we call + * when we want to rewrite the configuration files describing ACLs and + * in order to show users with ACL LIST. */ +sds ACLDescribeUser(user *u) { + sds res = sdsempty(); + + /* Flags. */ + for (int j = 0; ACLUserFlags[j].flag; j++) { + /* Skip the allcommands and allkeys flags because they'll be emitted + * later as ~* and +@all. */ + if (ACLUserFlags[j].flag == USER_FLAG_ALLKEYS || + ACLUserFlags[j].flag == USER_FLAG_ALLCOMMANDS) continue; + if (u->flags & ACLUserFlags[j].flag) { + res = sdscat(res,ACLUserFlags[j].name); + res = sdscatlen(res," ",1); + } + } + + /* Passwords. */ + listIter li; + listNode *ln; + listRewind(u->passwords,&li); + while((ln = listNext(&li))) { + sds thispass = listNodeValue(ln); + res = sdscatlen(res,"#",1); + res = sdscatsds(res,thispass); + res = sdscatlen(res," ",1); + } + + /* Key patterns. */ + if (u->flags & USER_FLAG_ALLKEYS) { + res = sdscatlen(res,"~* ",3); + } else { + listRewind(u->patterns,&li); + while((ln = listNext(&li))) { + sds thispat = listNodeValue(ln); + res = sdscatlen(res,"~",1); + res = sdscatsds(res,thispat); + res = sdscatlen(res," ",1); + } + } + + /* Command rules. */ + sds rules = ACLDescribeUserCommandRules(u); + res = sdscatsds(res,rules); + sdsfree(rules); + return res; +} + +/* Get a command from the original command table, that is not affected + * by the command renaming operations: we base all the ACL work from that + * table, so that ACLs are valid regardless of command renaming. */ +struct redisCommand *ACLLookupCommand(const char *name) { + struct redisCommand *cmd; + sds sdsname = sdsnew(name); + cmd = dictFetchValue(server.orig_commands, sdsname); + sdsfree(sdsname); + return cmd; +} + +/* Flush the array of allowed subcommands for the specified user + * and command ID. */ +void ACLResetSubcommandsForCommand(user *u, unsigned long id) { + if (u->allowed_subcommands && u->allowed_subcommands[id]) { + for (int i = 0; u->allowed_subcommands[id][i]; i++) + sdsfree(u->allowed_subcommands[id][i]); + zfree(u->allowed_subcommands[id]); + u->allowed_subcommands[id] = NULL; + } +} + +/* Flush the entire table of subcommands. This is useful on +@all, -@all + * or similar to return back to the minimal memory usage (and checks to do) + * for the user. */ +void ACLResetSubcommands(user *u) { + if (u->allowed_subcommands == NULL) return; + for (int j = 0; j < USER_COMMAND_BITS_COUNT; j++) { + if (u->allowed_subcommands[j]) { + for (int i = 0; u->allowed_subcommands[j][i]; i++) + sdsfree(u->allowed_subcommands[j][i]); + zfree(u->allowed_subcommands[j]); + } + } + zfree(u->allowed_subcommands); + u->allowed_subcommands = NULL; +} + + +/* Add a subcommand to the list of subcommands for the user 'u' and + * the command id specified. */ +void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { + /* If this is the first subcommand to be configured for + * this user, we have to allocate the subcommands array. */ + if (u->allowed_subcommands == NULL) { + u->allowed_subcommands = zcalloc(USER_COMMAND_BITS_COUNT * + sizeof(sds*)); + } + + /* We also need to enlarge the allocation pointing to the + * null terminated SDS array, to make space for this one. + * To start check the current size, and while we are here + * make sure the subcommand is not already specified inside. */ + long items = 0; + if (u->allowed_subcommands[id]) { + while(u->allowed_subcommands[id][items]) { + /* If it's already here do not add it again. */ + if (!strcasecmp(u->allowed_subcommands[id][items],sub)) return; + items++; + } + } + + /* Now we can make space for the new item (and the null term). */ + items += 2; + u->allowed_subcommands[id] = zrealloc(u->allowed_subcommands[id], + sizeof(sds)*items); + u->allowed_subcommands[id][items-2] = sdsnew(sub); + u->allowed_subcommands[id][items-1] = NULL; +} + +/* Set user properties according to the string "op". The following + * is a description of what different strings will do: + * + * on Enable the user: it is possible to authenticate as this user. + * off Disable the user: it's no longer possible to authenticate + * with this user, however the already authenticated connections + * will still work. + * + Allow the execution of that command + * - Disallow the execution of that command + * +@ Allow the execution of all the commands in such category + * with valid categories are like @admin, @set, @sortedset, ... + * and so forth, see the full list in the server.c file where + * the Redis command table is described and defined. + * The special category @all means all the commands, but currently + * present in the server, and that will be loaded in the future + * via modules. + * +|subcommand Allow a specific subcommand of an otherwise + * disabled command. Note that this form is not + * allowed as negative like -DEBUG|SEGFAULT, but + * only additive starting with "+". + * allcommands Alias for +@all. Note that it implies the ability to execute + * all the future commands loaded via the modules system. + * nocommands Alias for -@all. + * ~ Add a pattern of keys that can be mentioned as part of + * commands. For instance ~* allows all the keys. The pattern + * is a glob-style pattern like the one of KEYS. + * It is possible to specify multiple patterns. + * allkeys Alias for ~* + * resetkeys Flush the list of allowed keys patterns. + * > Add this password to the list of valid password for the user. + * For example >mypass will add "mypass" to the list. + * This directive clears the "nopass" flag (see later). + * # Add this password hash to the list of valid hashes for + * the user. This is useful if you have previously computed + * the hash, and don't want to store it in plaintext. + * This directive clears the "nopass" flag (see later). + * < Remove this password from the list of valid passwords. + * ! Remove this hashed password from the list of valid passwords. + * This is useful when you want to remove a password just by + * hash without knowing its plaintext version at all. + * nopass All the set passwords of the user are removed, and the user + * is flagged as requiring no password: it means that every + * password will work against this user. If this directive is + * used for the default user, every new connection will be + * immediately authenticated with the default user without + * any explicit AUTH command required. Note that the "resetpass" + * directive will clear this condition. + * resetpass Flush the list of allowed passwords. Moreover removes the + * "nopass" status. After "resetpass" the user has no associated + * passwords and there is no way to authenticate without adding + * some password (or setting it as "nopass" later). + * reset Performs the following actions: resetpass, resetkeys, off, + * -@all. The user returns to the same state it has immediately + * after its creation. + * + * The 'op' string must be null terminated. The 'oplen' argument should + * specify the length of the 'op' string in case the caller requires to pass + * binary data (for instance the >password form may use a binary password). + * Otherwise the field can be set to -1 and the function will use strlen() + * to determine the length. + * + * The function returns C_OK if the action to perform was understood because + * the 'op' string made sense. Otherwise C_ERR is returned if the operation + * is unknown or has some syntax error. + * + * When an error is returned, errno is set to the following values: + * + * EINVAL: The specified opcode is not understood or the key pattern is + * invalid (contains non allowed characters). + * ENOENT: The command name or command category provided with + or - is not + * known. + * EBUSY: The subcommand you want to add is about a command that is currently + * fully added. + * EEXIST: You are adding a key pattern after "*" was already added. This is + * almost surely an error on the user side. + * ENODEV: The password you are trying to remove from the user does not exist. + * EBADMSG: The hash you are trying to add is not a valid hash. + */ +int ACLSetUser(user *u, const char *op, ssize_t oplen) { + if (oplen == -1) oplen = strlen(op); + if (oplen == 0) return C_OK; /* Empty string is a no-operation. */ + if (!strcasecmp(op,"on")) { + u->flags |= USER_FLAG_ENABLED; + u->flags &= ~USER_FLAG_DISABLED; + } else if (!strcasecmp(op,"off")) { + u->flags |= USER_FLAG_DISABLED; + u->flags &= ~USER_FLAG_ENABLED; + } else if (!strcasecmp(op,"allkeys") || + !strcasecmp(op,"~*")) + { + u->flags |= USER_FLAG_ALLKEYS; + listEmpty(u->patterns); + } else if (!strcasecmp(op,"resetkeys")) { + u->flags &= ~USER_FLAG_ALLKEYS; + listEmpty(u->patterns); + } else if (!strcasecmp(op,"allcommands") || + !strcasecmp(op,"+@all")) + { + memset(u->allowed_commands,255,sizeof(u->allowed_commands)); + u->flags |= USER_FLAG_ALLCOMMANDS; + ACLResetSubcommands(u); + } else if (!strcasecmp(op,"nocommands") || + !strcasecmp(op,"-@all")) + { + memset(u->allowed_commands,0,sizeof(u->allowed_commands)); + u->flags &= ~USER_FLAG_ALLCOMMANDS; + ACLResetSubcommands(u); + } else if (!strcasecmp(op,"nopass")) { + u->flags |= USER_FLAG_NOPASS; + listEmpty(u->passwords); + } else if (!strcasecmp(op,"resetpass")) { + u->flags &= ~USER_FLAG_NOPASS; + listEmpty(u->passwords); + } else if (op[0] == '>' || op[0] == '#') { + sds newpass; + if (op[0] == '>') { + newpass = ACLHashPassword((unsigned char*)op+1,oplen-1); + } else { + if (ACLCheckPasswordHash((unsigned char*)op+1,oplen-1) == C_ERR) { + errno = EBADMSG; + return C_ERR; + } + newpass = sdsnewlen(op+1,oplen-1); + } + + listNode *ln = listSearchKey(u->passwords,newpass); + /* Avoid re-adding the same password multiple times. */ + if (ln == NULL) + listAddNodeTail(u->passwords,newpass); + else + sdsfree(newpass); + u->flags &= ~USER_FLAG_NOPASS; + } else if (op[0] == '<' || op[0] == '!') { + sds delpass; + if (op[0] == '<') { + delpass = ACLHashPassword((unsigned char*)op+1,oplen-1); + } else { + if (ACLCheckPasswordHash((unsigned char*)op+1,oplen-1) == C_ERR) { + errno = EBADMSG; + return C_ERR; + } + delpass = sdsnewlen(op+1,oplen-1); + } + listNode *ln = listSearchKey(u->passwords,delpass); + sdsfree(delpass); + if (ln) { + listDelNode(u->passwords,ln); + } else { + errno = ENODEV; + return C_ERR; + } + } else if (op[0] == '~') { + if (u->flags & USER_FLAG_ALLKEYS) { + errno = EEXIST; + return C_ERR; + } + if (ACLStringHasSpaces(op+1,oplen-1)) { + errno = EINVAL; + return C_ERR; + } + sds newpat = sdsnewlen(op+1,oplen-1); + listNode *ln = listSearchKey(u->patterns,newpat); + /* Avoid re-adding the same pattern multiple times. */ + if (ln == NULL) + listAddNodeTail(u->patterns,newpat); + else + sdsfree(newpat); + u->flags &= ~USER_FLAG_ALLKEYS; + } else if (op[0] == '+' && op[1] != '@') { + if (strchr(op,'|') == NULL) { + if (ACLLookupCommand(op+1) == NULL) { + errno = ENOENT; + return C_ERR; + } + unsigned long id = ACLGetCommandID(op+1); + ACLSetUserCommandBit(u,id,1); + ACLResetSubcommandsForCommand(u,id); + } else { + /* Split the command and subcommand parts. */ + char *copy = zstrdup(op+1); + char *sub = strchr(copy,'|'); + sub[0] = '\0'; + sub++; + + /* Check if the command exists. We can't check the + * subcommand to see if it is valid. */ + if (ACLLookupCommand(copy) == NULL) { + zfree(copy); + errno = ENOENT; + return C_ERR; + } + + /* The subcommand cannot be empty, so things like DEBUG| + * are syntax errors of course. */ + if (strlen(sub) == 0) { + zfree(copy); + errno = EINVAL; + return C_ERR; + } + + /* The command should not be set right now in the command + * bitmap, because adding a subcommand of a fully added + * command is probably an error on the user side. */ + unsigned long id = ACLGetCommandID(copy); + if (ACLGetUserCommandBit(u,id) == 1) { + zfree(copy); + errno = EBUSY; + return C_ERR; + } + + /* Add the subcommand to the list of valid ones. */ + ACLAddAllowedSubcommand(u,id,sub); + + /* We have to clear the command bit so that we force the + * subcommand check. */ + ACLSetUserCommandBit(u,id,0); + zfree(copy); + } + } else if (op[0] == '-' && op[1] != '@') { + if (ACLLookupCommand(op+1) == NULL) { + errno = ENOENT; + return C_ERR; + } + unsigned long id = ACLGetCommandID(op+1); + ACLSetUserCommandBit(u,id,0); + ACLResetSubcommandsForCommand(u,id); + } else if ((op[0] == '+' || op[0] == '-') && op[1] == '@') { + int bitval = op[0] == '+' ? 1 : 0; + if (ACLSetUserCommandBitsForCategory(u,op+2,bitval) == C_ERR) { + errno = ENOENT; + return C_ERR; + } + } else if (!strcasecmp(op,"reset")) { + serverAssert(ACLSetUser(u,"resetpass",-1) == C_OK); + serverAssert(ACLSetUser(u,"resetkeys",-1) == C_OK); + serverAssert(ACLSetUser(u,"off",-1) == C_OK); + serverAssert(ACLSetUser(u,"-@all",-1) == C_OK); + } else { + errno = EINVAL; + return C_ERR; + } + return C_OK; +} + +/* Return a description of the error that occurred in ACLSetUser() according to + * the errno value set by the function on error. */ +char *ACLSetUserStringError(void) { + char *errmsg = "Wrong format"; + if (errno == ENOENT) + errmsg = "Unknown command or category name in ACL"; + else if (errno == EINVAL) + errmsg = "Syntax error"; + else if (errno == EBUSY) + errmsg = "Adding a subcommand of a command already fully " + "added is not allowed. Remove the command to start. " + "Example: -DEBUG +DEBUG|DIGEST"; + else if (errno == EEXIST) + errmsg = "Adding a pattern after the * pattern (or the " + "'allkeys' flag) is not valid and does not have any " + "effect. Try 'resetkeys' to start with an empty " + "list of patterns"; + else if (errno == ENODEV) + errmsg = "The password you are trying to remove from the user does " + "not exist"; + else if (errno == EBADMSG) + errmsg = "The password hash must be exactly 64 characters and contain " + "only lowercase hexadecimal characters"; + return errmsg; +} + +/* Initialize the default user, that will always exist for all the process + * lifetime. */ +void ACLInitDefaultUser(void) { + DefaultUser = ACLCreateUser("default",7); + ACLSetUser(DefaultUser,"+@all",-1); + ACLSetUser(DefaultUser,"~*",-1); + ACLSetUser(DefaultUser,"on",-1); + ACLSetUser(DefaultUser,"nopass",-1); +} + +/* Initialization of the ACL subsystem. */ +void ACLInit(void) { + Users = raxNew(); + UsersToLoad = listCreate(); + ACLLog = listCreate(); + ACLInitDefaultUser(); + server.requirepass = NULL; /* Only used for backward compatibility. */ +} + +/* Check the username and password pair and return C_OK if they are valid, + * otherwise C_ERR is returned and errno is set to: + * + * EINVAL: if the username-password do not match. + * ENONENT: if the specified user does not exist at all. + */ +int ACLCheckUserCredentials(robj *username, robj *password) { + user *u = ACLGetUserByName(username->ptr,sdslen(username->ptr)); + if (u == NULL) { + errno = ENOENT; + return C_ERR; + } + + /* Disabled users can't login. */ + if (u->flags & USER_FLAG_DISABLED) { + errno = EINVAL; + return C_ERR; + } + + /* If the user is configured to don't require any password, we + * are already fine here. */ + if (u->flags & USER_FLAG_NOPASS) return C_OK; + + /* Check all the user passwords for at least one to match. */ + listIter li; + listNode *ln; + listRewind(u->passwords,&li); + sds hashed = ACLHashPassword(password->ptr,sdslen(password->ptr)); + while((ln = listNext(&li))) { + sds thispass = listNodeValue(ln); + if (!time_independent_strcmp(hashed, thispass)) { + sdsfree(hashed); + return C_OK; + } + } + sdsfree(hashed); + + /* If we reached this point, no password matched. */ + errno = EINVAL; + return C_ERR; +} + +/* This is like ACLCheckUserCredentials(), however if the user/pass + * are correct, the connection is put in authenticated state and the + * connection user reference is populated. + * + * The return value is C_OK or C_ERR with the same meaning as + * ACLCheckUserCredentials(). */ +int ACLAuthenticateUser(client *c, robj *username, robj *password) { + if (ACLCheckUserCredentials(username,password) == C_OK) { + c->authenticated = 1; + c->user = ACLGetUserByName(username->ptr,sdslen(username->ptr)); + moduleNotifyUserChanged(c); + return C_OK; + } else { + addACLLogEntry(c,ACL_DENIED_AUTH,0,username->ptr); + return C_ERR; + } +} + +/* For ACL purposes, every user has a bitmap with the commands that such + * user is allowed to execute. In order to populate the bitmap, every command + * should have an assigned ID (that is used to index the bitmap). This function + * creates such an ID: it uses sequential IDs, reusing the same ID for the same + * command name, so that a command retains the same ID in case of modules that + * are unloaded and later reloaded. */ +unsigned long ACLGetCommandID(const char *cmdname) { + static rax *map = NULL; + static unsigned long nextid = 0; + + sds lowername = sdsnew(cmdname); + sdstolower(lowername); + if (map == NULL) map = raxNew(); + void *id = raxFind(map,(unsigned char*)lowername,sdslen(lowername)); + if (id != raxNotFound) { + sdsfree(lowername); + return (unsigned long)id; + } + raxInsert(map,(unsigned char*)lowername,strlen(lowername), + (void*)nextid,NULL); + sdsfree(lowername); + unsigned long thisid = nextid; + nextid++; + + /* We never assign the last bit in the user commands bitmap structure, + * this way we can later check if this bit is set, understanding if the + * current ACL for the user was created starting with a +@all to add all + * the possible commands and just subtracting other single commands or + * categories, or if, instead, the ACL was created just adding commands + * and command categories from scratch, not allowing future commands by + * default (loaded via modules). This is useful when rewriting the ACLs + * with ACL SAVE. */ + if (nextid == USER_COMMAND_BITS_COUNT-1) nextid++; + return thisid; +} + +/* Return an username by its name, or NULL if the user does not exist. */ +user *ACLGetUserByName(const char *name, size_t namelen) { + void *myuser = raxFind(Users,(unsigned char*)name,namelen); + if (myuser == raxNotFound) return NULL; + return myuser; +} + +/* Check if the command is ready to be executed in the client 'c', already + * referenced by c->cmd, and can be executed by this client according to the + * ACLs associated to the client user c->user. + * + * If the user can execute the command ACL_OK is returned, otherwise + * ACL_DENIED_CMD or ACL_DENIED_KEY is returned: the first in case the + * command cannot be executed because the user is not allowed to run such + * command, the second if the command is denied because the user is trying + * to access keys that are not among the specified patterns. */ +int ACLCheckCommandPerm(client *c, int *keyidxptr) { + user *u = c->user; + uint64_t id = c->cmd->id; + + /* If there is no associated user, the connection can run anything. */ + if (u == NULL) return ACL_OK; + + /* Check if the user can execute this command. */ + if (!(u->flags & USER_FLAG_ALLCOMMANDS) && + c->cmd->proc != authCommand) + { + /* If the bit is not set we have to check further, in case the + * command is allowed just with that specific subcommand. */ + if (ACLGetUserCommandBit(u,id) == 0) { + /* Check if the subcommand matches. */ + if (c->argc < 2 || + u->allowed_subcommands == NULL || + u->allowed_subcommands[id] == NULL) + { + return ACL_DENIED_CMD; + } + + long subid = 0; + while (1) { + if (u->allowed_subcommands[id][subid] == NULL) + return ACL_DENIED_CMD; + if (!strcasecmp(c->argv[1]->ptr, + u->allowed_subcommands[id][subid])) + break; /* Subcommand match found. Stop here. */ + subid++; + } + } + } + + /* Check if the user can execute commands explicitly touching the keys + * mentioned in the command arguments. */ + if (!(c->user->flags & USER_FLAG_ALLKEYS) && + (c->cmd->getkeys_proc || c->cmd->firstkey)) + { + int numkeys; + int *keyidx = getKeysFromCommand(c->cmd,c->argv,c->argc,&numkeys); + for (int j = 0; j < numkeys; j++) { + listIter li; + listNode *ln; + listRewind(u->patterns,&li); + + /* Test this key against every pattern. */ + int match = 0; + while((ln = listNext(&li))) { + sds pattern = listNodeValue(ln); + size_t plen = sdslen(pattern); + int idx = keyidx[j]; + if (stringmatchlen(pattern,plen,c->argv[idx]->ptr, + sdslen(c->argv[idx]->ptr),0)) + { + match = 1; + break; + } + } + if (!match) { + if (keyidxptr) *keyidxptr = keyidx[j]; + getKeysFreeResult(keyidx); + return ACL_DENIED_KEY; + } + } + getKeysFreeResult(keyidx); + } + + /* If we survived all the above checks, the user can execute the + * command. */ + return ACL_OK; +} + +/* ============================================================================= + * ACL loading / saving functions + * ==========================================================================*/ + +/* Given an argument vector describing a user in the form: + * + * user ... ACL rules and flags ... + * + * this function validates, and if the syntax is valid, appends + * the user definition to a list for later loading. + * + * The rules are tested for validity and if there obvious syntax errors + * the function returns C_ERR and does nothing, otherwise C_OK is returned + * and the user is appended to the list. + * + * Note that this function cannot stop in case of commands that are not found + * and, in that case, the error will be emitted later, because certain + * commands may be defined later once modules are loaded. + * + * When an error is detected and C_ERR is returned, the function populates + * by reference (if not set to NULL) the argc_err argument with the index + * of the argv vector that caused the error. */ +int ACLAppendUserForLoading(sds *argv, int argc, int *argc_err) { + if (argc < 2 || strcasecmp(argv[0],"user")) { + if (argc_err) *argc_err = 0; + return C_ERR; + } + + /* Try to apply the user rules in a fake user to see if they + * are actually valid. */ + user *fakeuser = ACLCreateUnlinkedUser(); + + for (int j = 2; j < argc; j++) { + if (ACLSetUser(fakeuser,argv[j],sdslen(argv[j])) == C_ERR) { + if (errno != ENOENT) { + ACLFreeUser(fakeuser); + if (argc_err) *argc_err = j; + return C_ERR; + } + } + } + + /* Rules look valid, let's append the user to the list. */ + sds *copy = zmalloc(sizeof(sds)*argc); + for (int j = 1; j < argc; j++) copy[j-1] = sdsdup(argv[j]); + copy[argc-1] = NULL; + listAddNodeTail(UsersToLoad,copy); + ACLFreeUser(fakeuser); + return C_OK; +} + +/* This function will load the configured users appended to the server + * configuration via ACLAppendUserForLoading(). On loading errors it will + * log an error and return C_ERR, otherwise C_OK will be returned. */ +int ACLLoadConfiguredUsers(void) { + listIter li; + listNode *ln; + listRewind(UsersToLoad,&li); + while ((ln = listNext(&li)) != NULL) { + sds *aclrules = listNodeValue(ln); + sds username = aclrules[0]; + + if (ACLStringHasSpaces(aclrules[0],sdslen(aclrules[0]))) { + serverLog(LL_WARNING,"Spaces not allowed in ACL usernames"); + return C_ERR; + } + + user *u = ACLCreateUser(username,sdslen(username)); + if (!u) { + u = ACLGetUserByName(username,sdslen(username)); + serverAssert(u != NULL); + ACLSetUser(u,"reset",-1); + } + + /* Load every rule defined for this user. */ + for (int j = 1; aclrules[j]; j++) { + if (ACLSetUser(u,aclrules[j],sdslen(aclrules[j])) != C_OK) { + char *errmsg = ACLSetUserStringError(); + serverLog(LL_WARNING,"Error loading ACL rule '%s' for " + "the user named '%s': %s", + aclrules[j],aclrules[0],errmsg); + return C_ERR; + } + } + + /* Having a disabled user in the configuration may be an error, + * warn about it without returning any error to the caller. */ + if (u->flags & USER_FLAG_DISABLED) { + serverLog(LL_NOTICE, "The user '%s' is disabled (there is no " + "'on' modifier in the user description). Make " + "sure this is not a configuration error.", + aclrules[0]); + } + } + return C_OK; +} + +/* This function loads the ACL from the specified filename: every line + * is validated and should be either empty or in the format used to specify + * users in the redis.conf configuration or in the ACL file, that is: + * + * user ... rules ... + * + * Note that this function considers comments starting with '#' as errors + * because the ACL file is meant to be rewritten, and comments would be + * lost after the rewrite. Yet empty lines are allowed to avoid being too + * strict. + * + * One important part of implementing ACL LOAD, that uses this function, is + * to avoid ending with broken rules if the ACL file is invalid for some + * reason, so the function will attempt to validate the rules before loading + * each user. For every line that will be found broken the function will + * collect an error message. + * + * IMPORTANT: If there is at least a single error, nothing will be loaded + * and the rules will remain exactly as they were. + * + * At the end of the process, if no errors were found in the whole file then + * NULL is returned. Otherwise an SDS string describing in a single line + * a description of all the issues found is returned. */ +sds ACLLoadFromFile(const char *filename) { + FILE *fp; + char buf[1024]; + + /* Open the ACL file. */ + if ((fp = fopen(filename,"r")) == NULL) { + sds errors = sdscatprintf(sdsempty(), + "Error loading ACLs, opening file '%s': %s", + filename, strerror(errno)); + return errors; + } + + /* Load the whole file as a single string in memory. */ + sds acls = sdsempty(); + while(fgets(buf,sizeof(buf),fp) != NULL) + acls = sdscat(acls,buf); + fclose(fp); + + /* Split the file into lines and attempt to load each line. */ + int totlines; + sds *lines, errors = sdsempty(); + lines = sdssplitlen(acls,strlen(acls),"\n",1,&totlines); + sdsfree(acls); + + /* We need a fake user to validate the rules before making changes + * to the real user mentioned in the ACL line. */ + user *fakeuser = ACLCreateUnlinkedUser(); + + /* We do all the loading in a fresh instance of the Users radix tree, + * so if there are errors loading the ACL file we can rollback to the + * old version. */ + rax *old_users = Users; + user *old_default_user = DefaultUser; + Users = raxNew(); + ACLInitDefaultUser(); + + /* Load each line of the file. */ + for (int i = 0; i < totlines; i++) { + sds *argv; + int argc; + int linenum = i+1; + + lines[i] = sdstrim(lines[i]," \t\r\n"); + + /* Skip blank lines */ + if (lines[i][0] == '\0') continue; + + /* Split into arguments */ + argv = sdssplitlen(lines[i],sdslen(lines[i])," ",1,&argc); + if (argv == NULL) { + errors = sdscatprintf(errors, + "%s:%d: unbalanced quotes in acl line. ", + server.acl_filename, linenum); + continue; + } + + /* Skip this line if the resulting command vector is empty. */ + if (argc == 0) { + sdsfreesplitres(argv,argc); + continue; + } + + /* The line should start with the "user" keyword. */ + if (strcmp(argv[0],"user") || argc < 2) { + errors = sdscatprintf(errors, + "%s:%d should start with user keyword followed " + "by the username. ", server.acl_filename, + linenum); + sdsfreesplitres(argv,argc); + continue; + } + + /* Spaces are not allowed in usernames. */ + if (ACLStringHasSpaces(argv[1],sdslen(argv[1]))) { + errors = sdscatprintf(errors, + "'%s:%d: username '%s' contains invalid characters. ", + server.acl_filename, linenum, argv[1]); + continue; + } + + /* Try to process the line using the fake user to validate if + * the rules are able to apply cleanly. At this stage we also + * trim trailing spaces, so that we don't have to handle that + * in ACLSetUser(). */ + ACLSetUser(fakeuser,"reset",-1); + int j; + for (j = 2; j < argc; j++) { + argv[j] = sdstrim(argv[j],"\t\r\n"); + if (ACLSetUser(fakeuser,argv[j],sdslen(argv[j])) != C_OK) { + char *errmsg = ACLSetUserStringError(); + errors = sdscatprintf(errors, + "%s:%d: %s. ", + server.acl_filename, linenum, errmsg); + continue; + } + } + + /* Apply the rule to the new users set only if so far there + * are no errors, otherwise it's useless since we are going + * to discard the new users set anyway. */ + if (sdslen(errors) != 0) { + sdsfreesplitres(argv,argc); + continue; + } + + /* We can finally lookup the user and apply the rule. If the + * user already exists we always reset it to start. */ + user *u = ACLCreateUser(argv[1],sdslen(argv[1])); + if (!u) { + u = ACLGetUserByName(argv[1],sdslen(argv[1])); + serverAssert(u != NULL); + ACLSetUser(u,"reset",-1); + } + + /* Note that the same rules already applied to the fake user, so + * we just assert that everything goes well: it should. */ + for (j = 2; j < argc; j++) + serverAssert(ACLSetUser(u,argv[j],sdslen(argv[j])) == C_OK); + + sdsfreesplitres(argv,argc); + } + + ACLFreeUser(fakeuser); + sdsfreesplitres(lines,totlines); + DefaultUser = old_default_user; /* This pointer must never change. */ + + /* Check if we found errors and react accordingly. */ + if (sdslen(errors) == 0) { + /* The default user pointer is referenced in different places: instead + * of replacing such occurrences it is much simpler to copy the new + * default user configuration in the old one. */ + user *new = ACLGetUserByName("default",7); + serverAssert(new != NULL); + ACLCopyUser(DefaultUser,new); + ACLFreeUser(new); + raxInsert(Users,(unsigned char*)"default",7,DefaultUser,NULL); + raxRemove(old_users,(unsigned char*)"default",7,NULL); + ACLFreeUsersSet(old_users); + sdsfree(errors); + return NULL; + } else { + ACLFreeUsersSet(Users); + Users = old_users; + errors = sdscat(errors,"WARNING: ACL errors detected, no change to the previously active ACL rules was performed"); + return errors; + } +} + +/* Generate a copy of the ACLs currently in memory in the specified filename. + * Returns C_OK on success or C_ERR if there was an error during the I/O. + * When C_ERR is returned a log is produced with hints about the issue. */ +int ACLSaveToFile(const char *filename) { + sds acl = sdsempty(); + int fd = -1; + sds tmpfilename = NULL; + int retval = C_ERR; + + /* Let's generate an SDS string containing the new version of the + * ACL file. */ + raxIterator ri; + raxStart(&ri,Users); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + user *u = ri.data; + /* Return information in the configuration file format. */ + sds user = sdsnew("user "); + user = sdscatsds(user,u->name); + user = sdscatlen(user," ",1); + sds descr = ACLDescribeUser(u); + user = sdscatsds(user,descr); + sdsfree(descr); + acl = sdscatsds(acl,user); + acl = sdscatlen(acl,"\n",1); + sdsfree(user); + } + raxStop(&ri); + + /* Create a temp file with the new content. */ + tmpfilename = sdsnew(filename); + tmpfilename = sdscatfmt(tmpfilename,".tmp-%i-%I", + (int)getpid(),(int)mstime()); + if ((fd = open(tmpfilename,O_WRONLY|O_CREAT,0644)) == -1) { + serverLog(LL_WARNING,"Opening temp ACL file for ACL SAVE: %s", + strerror(errno)); + goto cleanup; + } + + /* Write it. */ + if (write(fd,acl,sdslen(acl)) != (ssize_t)sdslen(acl)) { + serverLog(LL_WARNING,"Writing ACL file for ACL SAVE: %s", + strerror(errno)); + goto cleanup; + } + close(fd); fd = -1; + + /* Let's replace the new file with the old one. */ + if (rename(tmpfilename,filename) == -1) { + serverLog(LL_WARNING,"Renaming ACL file for ACL SAVE: %s", + strerror(errno)); + goto cleanup; + } + sdsfree(tmpfilename); tmpfilename = NULL; + retval = C_OK; /* If we reached this point, everything is fine. */ + +cleanup: + if (fd != -1) close(fd); + if (tmpfilename) unlink(tmpfilename); + sdsfree(tmpfilename); + sdsfree(acl); + return retval; +} + +/* This function is called once the server is already running, modules are + * loaded, and we are ready to start, in order to load the ACLs either from + * the pending list of users defined in redis.conf, or from the ACL file. + * The function will just exit with an error if the user is trying to mix + * both the loading methods. */ +void ACLLoadUsersAtStartup(void) { + if (server.acl_filename[0] != '\0' && listLength(UsersToLoad) != 0) { + serverLog(LL_WARNING, + "Configuring Redis with users defined in redis.conf and at " + "the same setting an ACL file path is invalid. This setup " + "is very likely to lead to configuration errors and security " + "holes, please define either an ACL file or declare users " + "directly in your redis.conf, but not both."); + exit(1); + } + + if (ACLLoadConfiguredUsers() == C_ERR) { + serverLog(LL_WARNING, + "Critical error while loading ACLs. Exiting."); + exit(1); + } + + if (server.acl_filename[0] != '\0') { + sds errors = ACLLoadFromFile(server.acl_filename); + if (errors) { + serverLog(LL_WARNING, + "Aborting Redis startup because of ACL errors: %s", errors); + sdsfree(errors); + exit(1); + } + } +} + +/* ============================================================================= + * ACL log + * ==========================================================================*/ + +#define ACL_LOG_CTX_TOPLEVEL 0 +#define ACL_LOG_CTX_LUA 1 +#define ACL_LOG_CTX_MULTI 2 +#define ACL_LOG_GROUPING_MAX_TIME_DELTA 60000 + +/* This structure defines an entry inside the ACL log. */ +typedef struct ACLLogEntry { + uint64_t count; /* Number of times this happened recently. */ + int reason; /* Reason for denying the command. ACL_DENIED_*. */ + int context; /* Toplevel, Lua or MULTI/EXEC? ACL_LOG_CTX_*. */ + sds object; /* The key name or command name. */ + sds username; /* User the client is authenticated with. */ + mstime_t ctime; /* Milliseconds time of last update to this entry. */ + sds cinfo; /* Client info (last client if updated). */ +} ACLLogEntry; + +/* This function will check if ACL entries 'a' and 'b' are similar enough + * that we should actually update the existing entry in our ACL log instead + * of creating a new one. */ +int ACLLogMatchEntry(ACLLogEntry *a, ACLLogEntry *b) { + if (a->reason != b->reason) return 0; + if (a->context != b->context) return 0; + mstime_t delta = a->ctime - b->ctime; + if (delta < 0) delta = -delta; + if (delta > ACL_LOG_GROUPING_MAX_TIME_DELTA) return 0; + if (sdscmp(a->object,b->object) != 0) return 0; + if (sdscmp(a->username,b->username) != 0) return 0; + return 1; +} + +/* Release an ACL log entry. */ +void ACLFreeLogEntry(void *leptr) { + ACLLogEntry *le = leptr; + sdsfree(le->object); + sdsfree(le->username); + sdsfree(le->cinfo); + zfree(le); +} + +/* Adds a new entry in the ACL log, making sure to delete the old entry + * if we reach the maximum length allowed for the log. This function attempts + * to find similar entries in the current log in order to bump the counter of + * the log entry instead of creating many entries for very similar ACL + * rules issues. + * + * The keypos argument is only used when the reason is ACL_DENIED_KEY, since + * it allows the function to log the key name that caused the problem. + * Similarly the username is only passed when we failed to authenticate the + * user with AUTH or HELLO, for the ACL_DENIED_AUTH reason. Otherwise + * it will just be NULL. + */ +void addACLLogEntry(client *c, int reason, int keypos, sds username) { + /* Create a new entry. */ + struct ACLLogEntry *le = zmalloc(sizeof(*le)); + le->count = 1; + le->reason = reason; + le->username = sdsdup(reason == ACL_DENIED_AUTH ? username : c->user->name); + le->ctime = mstime(); + + switch(reason) { + case ACL_DENIED_CMD: le->object = sdsnew(c->cmd->name); break; + case ACL_DENIED_KEY: le->object = sdsnew(c->argv[keypos]->ptr); break; + case ACL_DENIED_AUTH: le->object = sdsnew(c->argv[0]->ptr); break; + default: le->object = sdsempty(); + } + + client *realclient = c; + if (realclient->flags & CLIENT_LUA) realclient = server.lua_caller; + + le->cinfo = catClientInfoString(sdsempty(),realclient); + if (c->flags & CLIENT_MULTI) { + le->context = ACL_LOG_CTX_MULTI; + } else if (c->flags & CLIENT_LUA) { + le->context = ACL_LOG_CTX_LUA; + } else { + le->context = ACL_LOG_CTX_TOPLEVEL; + } + + /* Try to match this entry with past ones, to see if we can just + * update an existing entry instead of creating a new one. */ + long toscan = 10; /* Do a limited work trying to find duplicated. */ + listIter li; + listNode *ln; + listRewind(ACLLog,&li); + ACLLogEntry *match = NULL; + while (toscan-- && (ln = listNext(&li)) != NULL) { + ACLLogEntry *current = listNodeValue(ln); + if (ACLLogMatchEntry(current,le)) { + match = current; + listDelNode(ACLLog,ln); + listAddNodeHead(ACLLog,current); + break; + } + } + + /* If there is a match update the entry, otherwise add it as a + * new one. */ + if (match) { + /* We update a few fields of the existing entry and bump the + * counter of events for this entry. */ + sdsfree(match->cinfo); + match->cinfo = le->cinfo; + match->ctime = le->ctime; + match->count++; + + /* Release the old entry. */ + le->cinfo = NULL; + ACLFreeLogEntry(le); + } else { + /* Add it to our list of entires. We'll have to trim the list + * to its maximum size. */ + listAddNodeHead(ACLLog, le); + while(listLength(ACLLog) > server.acllog_max_len) { + listNode *ln = listLast(ACLLog); + ACLLogEntry *le = listNodeValue(ln); + ACLFreeLogEntry(le); + listDelNode(ACLLog,ln); + } + } +} + +/* ============================================================================= + * ACL related commands + * ==========================================================================*/ + +/* ACL -- show and modify the configuration of ACL users. + * ACL HELP + * ACL LOAD + * ACL SAVE + * ACL LIST + * ACL USERS + * ACL CAT [] + * ACL SETUSER ... acl rules ... + * ACL DELUSER [...] + * ACL GETUSER + * ACL GENPASS [] + * ACL WHOAMI + * ACL LOG [ | RESET] + */ +void aclCommand(client *c) { + char *sub = c->argv[1]->ptr; + if (!strcasecmp(sub,"setuser") && c->argc >= 3) { + sds username = c->argv[2]->ptr; + /* Check username validity. */ + if (ACLStringHasSpaces(username,sdslen(username))) { + addReplyErrorFormat(c, + "Usernames can't contain spaces or null characters"); + return; + } + + /* Create a temporary user to validate and stage all changes against + * before applying to an existing user or creating a new user. If all + * arguments are valid the user parameters will all be applied together. + * If there are any errors then none of the changes will be applied. */ + user *tempu = ACLCreateUnlinkedUser(); + user *u = ACLGetUserByName(username,sdslen(username)); + if (u) ACLCopyUser(tempu, u); + + for (int j = 3; j < c->argc; j++) { + if (ACLSetUser(tempu,c->argv[j]->ptr,sdslen(c->argv[j]->ptr)) != C_OK) { + char *errmsg = ACLSetUserStringError(); + addReplyErrorFormat(c, + "Error in ACL SETUSER modifier '%s': %s", + (char*)c->argv[j]->ptr, errmsg); + + ACLFreeUser(tempu); + return; + } + } + + /* Overwrite the user with the temporary user we modified above. */ + if (!u) u = ACLCreateUser(username,sdslen(username)); + serverAssert(u != NULL); + ACLCopyUser(u, tempu); + ACLFreeUser(tempu); + addReply(c,shared.ok); + } else if (!strcasecmp(sub,"deluser") && c->argc >= 3) { + int deleted = 0; + for (int j = 2; j < c->argc; j++) { + sds username = c->argv[j]->ptr; + if (!strcmp(username,"default")) { + addReplyError(c,"The 'default' user cannot be removed"); + return; + } + } + + for (int j = 2; j < c->argc; j++) { + sds username = c->argv[j]->ptr; + user *u; + if (raxRemove(Users,(unsigned char*)username, + sdslen(username), + (void**)&u)) + { + ACLFreeUserAndKillClients(u); + deleted++; + } + } + addReplyLongLong(c,deleted); + } else if (!strcasecmp(sub,"getuser") && c->argc == 3) { + user *u = ACLGetUserByName(c->argv[2]->ptr,sdslen(c->argv[2]->ptr)); + if (u == NULL) { + addReplyNull(c); + return; + } + + addReplyMapLen(c,4); + + /* Flags */ + addReplyBulkCString(c,"flags"); + void *deflen = addReplyDeferredLen(c); + int numflags = 0; + for (int j = 0; ACLUserFlags[j].flag; j++) { + if (u->flags & ACLUserFlags[j].flag) { + addReplyBulkCString(c,ACLUserFlags[j].name); + numflags++; + } + } + setDeferredSetLen(c,deflen,numflags); + + /* Passwords */ + addReplyBulkCString(c,"passwords"); + addReplyArrayLen(c,listLength(u->passwords)); + listIter li; + listNode *ln; + listRewind(u->passwords,&li); + while((ln = listNext(&li))) { + sds thispass = listNodeValue(ln); + addReplyBulkCBuffer(c,thispass,sdslen(thispass)); + } + + /* Commands */ + addReplyBulkCString(c,"commands"); + sds cmddescr = ACLDescribeUserCommandRules(u); + addReplyBulkSds(c,cmddescr); + + /* Key patterns */ + addReplyBulkCString(c,"keys"); + if (u->flags & USER_FLAG_ALLKEYS) { + addReplyArrayLen(c,1); + addReplyBulkCBuffer(c,"*",1); + } else { + addReplyArrayLen(c,listLength(u->patterns)); + listIter li; + listNode *ln; + listRewind(u->patterns,&li); + while((ln = listNext(&li))) { + sds thispat = listNodeValue(ln); + addReplyBulkCBuffer(c,thispat,sdslen(thispat)); + } + } + } else if ((!strcasecmp(sub,"list") || !strcasecmp(sub,"users")) && + c->argc == 2) + { + int justnames = !strcasecmp(sub,"users"); + addReplyArrayLen(c,raxSize(Users)); + raxIterator ri; + raxStart(&ri,Users); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + user *u = ri.data; + if (justnames) { + addReplyBulkCBuffer(c,u->name,sdslen(u->name)); + } else { + /* Return information in the configuration file format. */ + sds config = sdsnew("user "); + config = sdscatsds(config,u->name); + config = sdscatlen(config," ",1); + sds descr = ACLDescribeUser(u); + config = sdscatsds(config,descr); + sdsfree(descr); + addReplyBulkSds(c,config); + } + } + raxStop(&ri); + } else if (!strcasecmp(sub,"whoami") && c->argc == 2) { + if (c->user != NULL) { + addReplyBulkCBuffer(c,c->user->name,sdslen(c->user->name)); + } else { + addReplyNull(c); + } + } else if (server.acl_filename[0] == '\0' && + (!strcasecmp(sub,"load") || !strcasecmp(sub,"save"))) + { + addReplyError(c,"This Redis instance is not configured to use an ACL file. You may want to specify users via the ACL SETUSER command and then issue a CONFIG REWRITE (assuming you have a Redis configuration file set) in order to store users in the Redis configuration."); + return; + } else if (!strcasecmp(sub,"load") && c->argc == 2) { + sds errors = ACLLoadFromFile(server.acl_filename); + if (errors == NULL) { + addReply(c,shared.ok); + } else { + addReplyError(c,errors); + sdsfree(errors); + } + } else if (!strcasecmp(sub,"save") && c->argc == 2) { + if (ACLSaveToFile(server.acl_filename) == C_OK) { + addReply(c,shared.ok); + } else { + addReplyError(c,"There was an error trying to save the ACLs. " + "Please check the server logs for more " + "information"); + } + } else if (!strcasecmp(sub,"cat") && c->argc == 2) { + void *dl = addReplyDeferredLen(c); + int j; + for (j = 0; ACLCommandCategories[j].flag != 0; j++) + addReplyBulkCString(c,ACLCommandCategories[j].name); + setDeferredArrayLen(c,dl,j); + } else if (!strcasecmp(sub,"cat") && c->argc == 3) { + uint64_t cflag = ACLGetCommandCategoryFlagByName(c->argv[2]->ptr); + if (cflag == 0) { + addReplyErrorFormat(c, "Unknown category '%s'", (char*)c->argv[2]->ptr); + return; + } + int arraylen = 0; + void *dl = addReplyDeferredLen(c); + dictIterator *di = dictGetIterator(server.orig_commands); + dictEntry *de; + while ((de = dictNext(di)) != NULL) { + struct redisCommand *cmd = dictGetVal(de); + if (cmd->flags & CMD_MODULE) continue; + if (cmd->flags & cflag) { + addReplyBulkCString(c,cmd->name); + arraylen++; + } + } + dictReleaseIterator(di); + setDeferredArrayLen(c,dl,arraylen); + } else if (!strcasecmp(sub,"genpass") && (c->argc == 2 || c->argc == 3)) { + #define GENPASS_MAX_BITS 4096 + char pass[GENPASS_MAX_BITS/8*2]; /* Hex representation. */ + long bits = 256; /* By default generate 256 bits passwords. */ + + if (c->argc == 3 && getLongFromObjectOrReply(c,c->argv[2],&bits,NULL) + != C_OK) return; + + if (bits <= 0 || bits > GENPASS_MAX_BITS) { + addReplyErrorFormat(c, + "ACL GENPASS argument must be the number of " + "bits for the output password, a positive number " + "up to %d",GENPASS_MAX_BITS); + return; + } + + long chars = (bits+3)/4; /* Round to number of characters to emit. */ + getRandomHexChars(pass,chars); + addReplyBulkCBuffer(c,pass,chars); + } else if (!strcasecmp(sub,"log") && (c->argc == 2 || c->argc ==3)) { + long count = 10; /* Number of entries to emit by default. */ + + /* Parse the only argument that LOG may have: it could be either + * the number of entries the user wants to display, or alternatively + * the "RESET" command in order to flush the old entries. */ + if (c->argc == 3) { + if (!strcasecmp(c->argv[2]->ptr,"reset")) { + listSetFreeMethod(ACLLog,ACLFreeLogEntry); + listEmpty(ACLLog); + listSetFreeMethod(ACLLog,NULL); + addReply(c,shared.ok); + return; + } else if (getLongFromObjectOrReply(c,c->argv[2],&count,NULL) + != C_OK) + { + return; + } + if (count < 0) count = 0; + } + + /* Fix the count according to the number of entries we got. */ + if ((size_t)count > listLength(ACLLog)) + count = listLength(ACLLog); + + addReplyArrayLen(c,count); + listIter li; + listNode *ln; + listRewind(ACLLog,&li); + mstime_t now = mstime(); + while (count-- && (ln = listNext(&li)) != NULL) { + ACLLogEntry *le = listNodeValue(ln); + addReplyMapLen(c,7); + addReplyBulkCString(c,"count"); + addReplyLongLong(c,le->count); + + addReplyBulkCString(c,"reason"); + char *reasonstr; + switch(le->reason) { + case ACL_DENIED_CMD: reasonstr="command"; break; + case ACL_DENIED_KEY: reasonstr="key"; break; + case ACL_DENIED_AUTH: reasonstr="auth"; break; + default: reasonstr="unknown"; + } + addReplyBulkCString(c,reasonstr); + + addReplyBulkCString(c,"context"); + char *ctxstr; + switch(le->context) { + case ACL_LOG_CTX_TOPLEVEL: ctxstr="toplevel"; break; + case ACL_LOG_CTX_MULTI: ctxstr="multi"; break; + case ACL_LOG_CTX_LUA: ctxstr="lua"; break; + default: ctxstr="unknown"; + } + addReplyBulkCString(c,ctxstr); + + addReplyBulkCString(c,"object"); + addReplyBulkCBuffer(c,le->object,sdslen(le->object)); + addReplyBulkCString(c,"username"); + addReplyBulkCBuffer(c,le->username,sdslen(le->username)); + addReplyBulkCString(c,"age-seconds"); + double age = (double)(now - le->ctime)/1000; + addReplyDouble(c,age); + addReplyBulkCString(c,"client-info"); + addReplyBulkCBuffer(c,le->cinfo,sdslen(le->cinfo)); + } + } else if (!strcasecmp(sub,"help")) { + const char *help[] = { +"LOAD -- Reload users from the ACL file.", +"SAVE -- Save the current config to the ACL file.", +"LIST -- Show user details in config file format.", +"USERS -- List all the registered usernames.", +"SETUSER [attribs ...] -- Create or modify a user.", +"GETUSER -- Get the user details.", +"DELUSER [...] -- Delete a list of users.", +"CAT -- List available categories.", +"CAT -- List commands inside category.", +"GENPASS [] -- Generate a secure user password.", +"WHOAMI -- Return the current connection username.", +"LOG [ | RESET] -- Show the ACL log entries.", +NULL + }; + addReplyHelp(c,help); + } else { + addReplySubcommandSyntaxError(c); + } +} + +void addReplyCommandCategories(client *c, struct redisCommand *cmd) { + int flagcount = 0; + void *flaglen = addReplyDeferredLen(c); + for (int j = 0; ACLCommandCategories[j].flag != 0; j++) { + if (cmd->flags & ACLCommandCategories[j].flag) { + addReplyStatusFormat(c, "@%s", ACLCommandCategories[j].name); + flagcount++; + } + } + setDeferredSetLen(c, flaglen, flagcount); +} + +/* AUTH + * AUTH (Redis >= 6.0 form) + * + * When the user is omitted it means that we are trying to authenticate + * against the default user. */ +void authCommand(client *c) { + /* Only two or three argument forms are allowed. */ + if (c->argc > 3) { + addReply(c,shared.syntaxerr); + return; + } + + /* Handle the two different forms here. The form with two arguments + * will just use "default" as username. */ + robj *username, *password; + if (c->argc == 2) { + /* Mimic the old behavior of giving an error for the two commands + * from if no password is configured. */ + if (DefaultUser->flags & USER_FLAG_NOPASS) { + addReplyError(c,"AUTH called without any password " + "configured for the default user. Are you sure " + "your configuration is correct?"); + return; + } + + username = createStringObject("default",7); + password = c->argv[1]; + } else { + username = c->argv[1]; + password = c->argv[2]; + } + + if (ACLAuthenticateUser(c,username,password) == C_OK) { + addReply(c,shared.ok); + } else { + addReplyError(c,"-WRONGPASS invalid username-password pair"); + } + + /* Free the "default" string object we created for the two + * arguments form. */ + if (c->argc == 2) decrRefCount(username); +} + diff --git a/redis.submodule/src/adlist.c b/redis.submodule/src/adlist.c index ec5f8bb..0fedc07 100644 --- a/redis.submodule/src/adlist.c +++ b/redis.submodule/src/adlist.c @@ -327,12 +327,11 @@ listNode *listIndex(list *list, long index) { } /* Rotate the list removing the tail node and inserting it to the head. */ -void listRotate(list *list) { - listNode *tail = list->tail; - +void listRotateTailToHead(list *list) { if (listLength(list) <= 1) return; /* Detach current tail */ + listNode *tail = list->tail; list->tail = tail->prev; list->tail->next = NULL; /* Move it as head */ @@ -342,6 +341,21 @@ void listRotate(list *list) { list->head = tail; } +/* Rotate the list removing the head node and inserting it to the tail. */ +void listRotateHeadToTail(list *list) { + if (listLength(list) <= 1) return; + + listNode *head = list->head; + /* Detach current head */ + list->head = head->next; + list->head->prev = NULL; + /* Move it as tail */ + list->tail->next = head; + head->next = NULL; + head->prev = list->tail; + list->tail = head; +} + /* Add all the elements of the list 'o' at the end of the * list 'l'. The list 'other' remains empty but otherwise valid. */ void listJoin(list *l, list *o) { diff --git a/redis.submodule/src/adlist.h b/redis.submodule/src/adlist.h index c954fac..dd8a8d6 100644 --- a/redis.submodule/src/adlist.h +++ b/redis.submodule/src/adlist.h @@ -66,7 +66,7 @@ typedef struct list { #define listSetMatchMethod(l,m) ((l)->match = (m)) #define listGetDupMethod(l) ((l)->dup) -#define listGetFree(l) ((l)->free) +#define listGetFreeMethod(l) ((l)->free) #define listGetMatchMethod(l) ((l)->match) /* Prototypes */ @@ -85,7 +85,8 @@ listNode *listSearchKey(list *list, void *key); listNode *listIndex(list *list, long index); void listRewind(list *list, listIter *li); void listRewindTail(list *list, listIter *li); -void listRotate(list *list); +void listRotateTailToHead(list *list); +void listRotateHeadToTail(list *list); void listJoin(list *l, list *o); /* Directions for iterators */ diff --git a/redis.submodule/src/ae.c b/redis.submodule/src/ae.c index 1ea6715..689a27d 100644 --- a/redis.submodule/src/ae.c +++ b/redis.submodule/src/ae.c @@ -76,6 +76,7 @@ aeEventLoop *aeCreateEventLoop(int setsize) { eventLoop->maxfd = -1; eventLoop->beforesleep = NULL; eventLoop->aftersleep = NULL; + eventLoop->flags = 0; if (aeApiCreate(eventLoop) == -1) goto err; /* Events with mask == AE_NONE are not set. So let's initialize the * vector with it. */ @@ -97,6 +98,14 @@ int aeGetSetSize(aeEventLoop *eventLoop) { return eventLoop->setsize; } +/* Tells the next iteration/s of the event processing to set timeout of 0. */ +void aeSetDontWait(aeEventLoop *eventLoop, int noWait) { + if (noWait) + eventLoop->flags |= AE_DONT_WAIT; + else + eventLoop->flags &= ~AE_DONT_WAIT; +} + /* Resize the maximum set size of the event loop. * If the requested set size is smaller than the current set size, but * there is already a file descriptor in use that is >= the requested @@ -126,6 +135,14 @@ void aeDeleteEventLoop(aeEventLoop *eventLoop) { aeApiFree(eventLoop); zfree(eventLoop->events); zfree(eventLoop->fired); + + /* Free the time events list. */ + aeTimeEvent *next_te, *te = eventLoop->timeEventHead; + while (te) { + next_te = te->next; + zfree(te); + te = next_te; + } zfree(eventLoop); } @@ -221,6 +238,7 @@ long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, te->clientData = clientData; te->prev = NULL; te->next = eventLoop->timeEventHead; + te->refcount = 0; if (te->next) te->next->prev = te; eventLoop->timeEventHead = te; @@ -299,6 +317,13 @@ static int processTimeEvents(aeEventLoop *eventLoop) { /* Remove events scheduled for deletion. */ if (te->id == AE_DELETED_EVENT_ID) { aeTimeEvent *next = te->next; + /* If a reference exists for this timer event, + * don't free it. This is currently incremented + * for recursive timerProc calls */ + if (te->refcount) { + te = next; + continue; + } if (te->prev) te->prev->next = te->next; else @@ -328,7 +353,9 @@ static int processTimeEvents(aeEventLoop *eventLoop) { int retval; id = te->id; + te->refcount++; retval = te->timeProc(eventLoop, id, te->clientData); + te->refcount--; processed++; if (retval != AE_NOMORE) { aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms); @@ -351,8 +378,9 @@ static int processTimeEvents(aeEventLoop *eventLoop) { * if flags has AE_FILE_EVENTS set, file events are processed. * if flags has AE_TIME_EVENTS set, time events are processed. * if flags has AE_DONT_WAIT set the function returns ASAP until all - * if flags has AE_CALL_AFTER_SLEEP set, the aftersleep callback is called. * the events that's possible to process without to wait are processed. + * if flags has AE_CALL_AFTER_SLEEP set, the aftersleep callback is called. + * if flags has AE_CALL_BEFORE_SLEEP set, the beforesleep callback is called. * * The function returns the number of events processed. */ int aeProcessEvents(aeEventLoop *eventLoop, int flags) @@ -406,6 +434,14 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) } } + if (eventLoop->flags & AE_DONT_WAIT) { + tv.tv_sec = tv.tv_usec = 0; + tvp = &tv; + } + + if (eventLoop->beforesleep != NULL && flags & AE_CALL_BEFORE_SLEEP) + eventLoop->beforesleep(eventLoop); + /* Call the multiplexing API, will return only on timeout or when * some event fires. */ numevents = aeApiPoll(eventLoop, tvp); @@ -442,6 +478,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) if (!invert && fe->mask & mask & AE_READABLE) { fe->rfileProc(eventLoop,fd,fe->clientData,mask); fired++; + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ } /* Fire the writable event. */ @@ -454,8 +491,11 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) /* If we have to invert the call, fire the readable event now * after the writable one. */ - if (invert && fe->mask & mask & AE_READABLE) { - if (!fired || fe->wfileProc != fe->rfileProc) { + if (invert) { + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ + if ((fe->mask & mask & AE_READABLE) && + (!fired || fe->wfileProc != fe->rfileProc)) + { fe->rfileProc(eventLoop,fd,fe->clientData,mask); fired++; } @@ -496,9 +536,9 @@ int aeWait(int fd, int mask, long long milliseconds) { void aeMain(aeEventLoop *eventLoop) { eventLoop->stop = 0; while (!eventLoop->stop) { - if (eventLoop->beforesleep != NULL) - eventLoop->beforesleep(eventLoop); - aeProcessEvents(eventLoop, AE_ALL_EVENTS|AE_CALL_AFTER_SLEEP); + aeProcessEvents(eventLoop, AE_ALL_EVENTS| + AE_CALL_BEFORE_SLEEP| + AE_CALL_AFTER_SLEEP); } } diff --git a/redis.submodule/src/ae.h b/redis.submodule/src/ae.h index 184fe3d..d1b7f34 100644 --- a/redis.submodule/src/ae.h +++ b/redis.submodule/src/ae.h @@ -47,11 +47,12 @@ things to disk before sending replies, and want to do that in a group fashion. */ -#define AE_FILE_EVENTS 1 -#define AE_TIME_EVENTS 2 +#define AE_FILE_EVENTS (1<<0) +#define AE_TIME_EVENTS (1<<1) #define AE_ALL_EVENTS (AE_FILE_EVENTS|AE_TIME_EVENTS) -#define AE_DONT_WAIT 4 -#define AE_CALL_AFTER_SLEEP 8 +#define AE_DONT_WAIT (1<<2) +#define AE_CALL_BEFORE_SLEEP (1<<3) +#define AE_CALL_AFTER_SLEEP (1<<4) #define AE_NOMORE -1 #define AE_DELETED_EVENT_ID -1 @@ -85,6 +86,8 @@ typedef struct aeTimeEvent { void *clientData; struct aeTimeEvent *prev; struct aeTimeEvent *next; + int refcount; /* refcount to prevent timer events from being + * freed in recursive time event calls. */ } aeTimeEvent; /* A fired event */ @@ -106,6 +109,7 @@ typedef struct aeEventLoop { void *apidata; /* This is used for polling API specific data */ aeBeforeSleepProc *beforesleep; aeBeforeSleepProc *aftersleep; + int flags; } aeEventLoop; /* Prototypes */ @@ -128,5 +132,6 @@ void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep void aeSetAfterSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *aftersleep); int aeGetSetSize(aeEventLoop *eventLoop); int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); +void aeSetDontWait(aeEventLoop *eventLoop, int noWait); #endif diff --git a/redis.submodule/src/ae_epoll.c b/redis.submodule/src/ae_epoll.c index 410aac7..fa19729 100644 --- a/redis.submodule/src/ae_epoll.c +++ b/redis.submodule/src/ae_epoll.c @@ -121,8 +121,8 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { if (e->events & EPOLLIN) mask |= AE_READABLE; if (e->events & EPOLLOUT) mask |= AE_WRITABLE; - if (e->events & EPOLLERR) mask |= AE_WRITABLE; - if (e->events & EPOLLHUP) mask |= AE_WRITABLE; + if (e->events & EPOLLERR) mask |= AE_WRITABLE|AE_READABLE; + if (e->events & EPOLLHUP) mask |= AE_WRITABLE|AE_READABLE; eventLoop->fired[j].fd = e->data.fd; eventLoop->fired[j].mask = mask; } diff --git a/redis.submodule/src/anet.c b/redis.submodule/src/anet.c index 2981fca..46ea7e1 100644 --- a/redis.submodule/src/anet.c +++ b/redis.submodule/src/anet.c @@ -193,6 +193,20 @@ int anetSendTimeout(char *err, int fd, long long ms) { return ANET_OK; } +/* Set the socket receive timeout (SO_RCVTIMEO socket option) to the specified + * number of milliseconds, or disable it if the 'ms' argument is zero. */ +int anetRecvTimeout(char *err, int fd, long long ms) { + struct timeval tv; + + tv.tv_sec = ms/1000; + tv.tv_usec = (ms%1000)*1000; + if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) == -1) { + anetSetError(err, "setsockopt SO_RCVTIMEO: %s", strerror(errno)); + return ANET_ERR; + } + return ANET_OK; +} + /* anetGenericResolve() is called by anetResolve() and anetResolveIP() to * do the actual work. It resolves the hostname "host" and set the string * representation of the IP address into the buffer pointed by "ipbuf". @@ -265,8 +279,8 @@ static int anetCreateSocket(char *err, int domain) { #define ANET_CONNECT_NONE 0 #define ANET_CONNECT_NONBLOCK 1 #define ANET_CONNECT_BE_BINDING 2 /* Best effort binding. */ -static int anetTcpGenericConnect(char *err, char *addr, int port, - char *source_addr, int flags) +static int anetTcpGenericConnect(char *err, const char *addr, int port, + const char *source_addr, int flags) { int s = ANET_ERR, rv; char portstr[6]; /* strlen("65535") + 1; */ @@ -345,31 +359,31 @@ static int anetTcpGenericConnect(char *err, char *addr, int port, } } -int anetTcpConnect(char *err, char *addr, int port) +int anetTcpConnect(char *err, const char *addr, int port) { return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONE); } -int anetTcpNonBlockConnect(char *err, char *addr, int port) +int anetTcpNonBlockConnect(char *err, const char *addr, int port) { return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONBLOCK); } -int anetTcpNonBlockBindConnect(char *err, char *addr, int port, - char *source_addr) +int anetTcpNonBlockBindConnect(char *err, const char *addr, int port, + const char *source_addr) { return anetTcpGenericConnect(err,addr,port,source_addr, ANET_CONNECT_NONBLOCK); } -int anetTcpNonBlockBestEffortBindConnect(char *err, char *addr, int port, - char *source_addr) +int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, + const char *source_addr) { return anetTcpGenericConnect(err,addr,port,source_addr, ANET_CONNECT_NONBLOCK|ANET_CONNECT_BE_BINDING); } -int anetUnixGenericConnect(char *err, char *path, int flags) +int anetUnixGenericConnect(char *err, const char *path, int flags) { int s; struct sockaddr_un sa; @@ -397,12 +411,12 @@ int anetUnixGenericConnect(char *err, char *path, int flags) return s; } -int anetUnixConnect(char *err, char *path) +int anetUnixConnect(char *err, const char *path) { return anetUnixGenericConnect(err,path,ANET_CONNECT_NONE); } -int anetUnixNonBlockConnect(char *err, char *path) +int anetUnixNonBlockConnect(char *err, const char *path) { return anetUnixGenericConnect(err,path,ANET_CONNECT_NONBLOCK); } diff --git a/redis.submodule/src/anet.h b/redis.submodule/src/anet.h index 7142f78..23f1964 100644 --- a/redis.submodule/src/anet.h +++ b/redis.submodule/src/anet.h @@ -49,12 +49,12 @@ #undef ip_len #endif -int anetTcpConnect(char *err, char *addr, int port); -int anetTcpNonBlockConnect(char *err, char *addr, int port); -int anetTcpNonBlockBindConnect(char *err, char *addr, int port, char *source_addr); -int anetTcpNonBlockBestEffortBindConnect(char *err, char *addr, int port, char *source_addr); -int anetUnixConnect(char *err, char *path); -int anetUnixNonBlockConnect(char *err, char *path); +int anetTcpConnect(char *err, const char *addr, int port); +int anetTcpNonBlockConnect(char *err, const char *addr, int port); +int anetTcpNonBlockBindConnect(char *err, const char *addr, int port, const char *source_addr); +int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, const char *source_addr); +int anetUnixConnect(char *err, const char *path); +int anetUnixNonBlockConnect(char *err, const char *path); int anetRead(int fd, char *buf, int count); int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len); int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len); @@ -70,6 +70,7 @@ int anetEnableTcpNoDelay(char *err, int fd); int anetDisableTcpNoDelay(char *err, int fd); int anetTcpKeepAlive(char *err, int fd); int anetSendTimeout(char *err, int fd, long long ms); +int anetRecvTimeout(char *err, int fd, long long ms); int anetPeerToString(int fd, char *ip, size_t ip_len, int *port); int anetKeepAlive(char *err, int fd, int interval); int anetSockName(int fd, char *ip, size_t ip_len, int *port); diff --git a/redis.submodule/src/aof.c b/redis.submodule/src/aof.c index 1014164..6f8e537 100644 --- a/redis.submodule/src/aof.c +++ b/redis.submodule/src/aof.c @@ -210,7 +210,7 @@ void aof_background_fsync(int fd) { } /* Kills an AOFRW child process if exists */ -static void killAppendOnlyChild(void) { +void killAppendOnlyChild(void) { int statloc; /* No AOFRW child? return. */ if (server.aof_child_pid == -1) return; @@ -227,6 +227,8 @@ static void killAppendOnlyChild(void) { server.aof_rewrite_time_start = -1; /* Close pipes used for IPC between the two processes. */ aofClosePipes(); + closeChildInfoPipe(); + updateDictResizePolicy(); } /* Called when the user switches from "appendonly yes" to "appendonly no" @@ -240,6 +242,7 @@ void stopAppendOnly(void) { server.aof_fd = -1; server.aof_selected_db = -1; server.aof_state = AOF_OFF; + server.aof_rewrite_scheduled = 0; killAppendOnlyChild(); } @@ -262,9 +265,9 @@ int startAppendOnly(void) { strerror(errno)); return C_ERR; } - if (server.rdb_child_pid != -1) { + if (hasActiveChildProcess() && server.aof_child_pid == -1) { server.aof_rewrite_scheduled = 1; - serverLog(LL_WARNING,"AOF was enabled but there is already a child process saving an RDB file on disk. An AOF background was scheduled to start when possible."); + serverLog(LL_WARNING,"AOF was enabled but there is already another background operation. An AOF background was scheduled to start when possible."); } else { /* If there is a pending AOF rewrite, we need to switch it off and * start a new one: the old one cannot be reused because it is not @@ -301,9 +304,7 @@ ssize_t aofWrite(int fd, const char *buf, size_t len) { nwritten = write(fd, buf, len); if (nwritten < 0) { - if (errno == EINTR) { - continue; - } + if (errno == EINTR) continue; return totwritten ? totwritten : -1; } @@ -385,6 +386,10 @@ void flushAppendOnlyFile(int force) { * there is much to do about the whole server stopping for power problems * or alike */ + if (server.aof_flush_sleep && sdslen(server.aof_buf)) { + usleep(server.aof_flush_sleep); + } + latencyStartMonitor(latency); nwritten = aofWrite(server.aof_fd,server.aof_buf,sdslen(server.aof_buf)); latencyEndMonitor(latency); @@ -395,7 +400,7 @@ void flushAppendOnlyFile(int force) { * useful for graphing / monitoring purposes. */ if (sync_in_progress) { latencyAddSampleIfNeeded("aof-write-pending-fsync",latency); - } else if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) { + } else if (hasActiveChildProcess()) { latencyAddSampleIfNeeded("aof-write-active-child",latency); } else { latencyAddSampleIfNeeded("aof-write-alone",latency); @@ -491,9 +496,8 @@ void flushAppendOnlyFile(int force) { try_fsync: /* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are * children doing I/O in the background. */ - if (server.aof_no_fsync_on_rewrite && - (server.aof_child_pid != -1 || server.rdb_child_pid != -1)) - return; + if (server.aof_no_fsync_on_rewrite && hasActiveChildProcess()) + return; /* Perform the fsync if needed. */ if (server.aof_fsync == AOF_FSYNC_ALWAYS) { @@ -607,19 +611,24 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a } else if (cmd->proc == setCommand && argc > 3) { int i; robj *exarg = NULL, *pxarg = NULL; - /* Translate SET [EX seconds][PX milliseconds] to SET and PEXPIREAT */ - buf = catAppendOnlyGenericCommand(buf,3,argv); for (i = 3; i < argc; i ++) { if (!strcasecmp(argv[i]->ptr, "ex")) exarg = argv[i+1]; if (!strcasecmp(argv[i]->ptr, "px")) pxarg = argv[i+1]; } serverAssert(!(exarg && pxarg)); - if (exarg) - buf = catAppendOnlyExpireAtCommand(buf,server.expireCommand,argv[1], - exarg); - if (pxarg) - buf = catAppendOnlyExpireAtCommand(buf,server.pexpireCommand,argv[1], - pxarg); + + if (exarg || pxarg) { + /* Translate SET [EX seconds][PX milliseconds] to SET and PEXPIREAT */ + buf = catAppendOnlyGenericCommand(buf,3,argv); + if (exarg) + buf = catAppendOnlyExpireAtCommand(buf,server.expireCommand,argv[1], + exarg); + if (pxarg) + buf = catAppendOnlyExpireAtCommand(buf,server.pexpireCommand,argv[1], + pxarg); + } else { + buf = catAppendOnlyGenericCommand(buf,argc,argv); + } } else { /* All the other commands don't need translation or need the * same translation already operated in the command vector @@ -649,11 +658,12 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a /* In Redis commands are always executed in the context of a client, so in * order to load the append only file we need to create a fake client. */ -struct client *createFakeClient(void) { +struct client *createAOFClient(void) { struct client *c = zmalloc(sizeof(*c)); selectDb(c,0); - c->fd = -1; + c->id = CLIENT_ID_AOF; /* So modules can identify it's the AOF client. */ + c->conn = NULL; c->name = NULL; c->querybuf = sdsempty(); c->querybuf_peak = 0; @@ -670,6 +680,8 @@ struct client *createFakeClient(void) { c->obuf_soft_limit_reached_time = 0; c->watched_keys = listCreate(); c->peerid = NULL; + c->resp = 2; + c->user = NULL; listSetFreeMethod(c->reply,freeClientReplyValue); listSetDupMethod(c->reply,dupClientReplyValue); initClientMultiState(c); @@ -724,8 +736,8 @@ int loadAppendOnlyFile(char *filename) { * to the same file we're about to read. */ server.aof_state = AOF_OFF; - fakeClient = createFakeClient(); - startLoading(fp); + fakeClient = createAOFClient(); + startLoadingFile(fp, filename, RDBFLAGS_AOF_PREAMBLE); /* Check if this AOF file has an RDB preamble. In that case we need to * load the RDB file and later continue loading the AOF tail. */ @@ -740,7 +752,7 @@ int loadAppendOnlyFile(char *filename) { serverLog(LL_NOTICE,"Reading RDB preamble from AOF file..."); if (fseek(fp,0,SEEK_SET) == -1) goto readerr; rioInitWithFile(&rdb,fp); - if (rdbLoadRio(&rdb,NULL,1) != C_OK) { + if (rdbLoadRio(&rdb,RDBFLAGS_AOF_PREAMBLE,NULL) != C_OK) { serverLog(LL_WARNING,"Error reading the RDB preamble of the AOF file, AOF loading aborted"); goto readerr; } else { @@ -761,6 +773,7 @@ int loadAppendOnlyFile(char *filename) { if (!(loops++ % 1000)) { loadingProgress(ftello(fp)); processEventsWhileBlocked(); + processModuleLoadingProgressEvent(1); } if (fgets(buf,sizeof(buf),fp) == NULL) { @@ -774,18 +787,26 @@ int loadAppendOnlyFile(char *filename) { argc = atoi(buf+1); if (argc < 1) goto fmterr; + /* Load the next command in the AOF as our fake client + * argv. */ argv = zmalloc(sizeof(robj*)*argc); fakeClient->argc = argc; fakeClient->argv = argv; for (j = 0; j < argc; j++) { - if (fgets(buf,sizeof(buf),fp) == NULL) { + /* Parse the argument len. */ + char *readres = fgets(buf,sizeof(buf),fp); + if (readres == NULL || buf[0] != '$') { fakeClient->argc = j; /* Free up to j-1. */ freeFakeClientArgv(fakeClient); - goto readerr; + if (readres == NULL) + goto readerr; + else + goto fmterr; } - if (buf[0] != '$') goto fmterr; len = strtol(buf+1,NULL,10); + + /* Read it into a string object. */ argsds = sdsnewlen(SDS_NOINIT,len); if (len && fread(argsds,len,1,fp) == 0) { sdsfree(argsds); @@ -794,10 +815,12 @@ int loadAppendOnlyFile(char *filename) { goto readerr; } argv[j] = createObject(OBJ_STRING,argsds); + + /* Discard CRLF. */ if (fread(buf,2,1,fp) == 0) { fakeClient->argc = j+1; /* Free up to j. */ freeFakeClientArgv(fakeClient); - goto readerr; /* discard CRLF */ + goto readerr; } } @@ -813,7 +836,7 @@ int loadAppendOnlyFile(char *filename) { if (cmd == server.multiCommand) valid_before_multi = valid_up_to; /* Run the command in the context of a fake client */ - fakeClient->cmd = cmd; + fakeClient->cmd = fakeClient->lastcmd = cmd; if (fakeClient->flags & CLIENT_MULTI && fakeClient->cmd->proc != execCommand) { @@ -834,6 +857,8 @@ int loadAppendOnlyFile(char *filename) { freeFakeClientArgv(fakeClient); fakeClient->cmd = NULL; if (server.aof_load_truncated) valid_up_to = ftello(fp); + if (server.key_load_delay) + usleep(server.key_load_delay); } /* This point can only be reached when EOF is reached without errors. @@ -851,7 +876,7 @@ int loadAppendOnlyFile(char *filename) { fclose(fp); freeFakeClient(fakeClient); server.aof_state = old_aof_state; - stopLoading(); + stopLoading(1); aofUpdateCurrentSize(); server.aof_rewrite_base_size = server.aof_current_size; server.aof_fsync_offset = server.aof_current_size; @@ -860,6 +885,7 @@ int loadAppendOnlyFile(char *filename) { readerr: /* Read error. If feof(fp) is true, fall through to unexpected EOF. */ if (!feof(fp)) { if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ + fclose(fp); serverLog(LL_WARNING,"Unrecoverable error reading the append only file: %s", strerror(errno)); exit(1); } @@ -890,11 +916,13 @@ int loadAppendOnlyFile(char *filename) { } } if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ + fclose(fp); serverLog(LL_WARNING,"Unexpected end of file reading the append only file. You can: 1) Make a backup of your AOF file, then use ./redis-check-aof --fix . 2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server."); exit(1); fmterr: /* Format error. */ if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ + fclose(fp); serverLog(LL_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix "); exit(1); } @@ -1127,7 +1155,7 @@ int rioWriteBulkStreamID(rio *r,streamID *id) { int retval; sds replyid = sdscatfmt(sdsempty(),"%U-%U",id->ms,id->seq); - if ((retval = rioWriteBulkString(r,replyid,sdslen(replyid))) == 0) return 0; + retval = rioWriteBulkString(r,replyid,sdslen(replyid)); sdsfree(replyid); return retval; } @@ -1189,12 +1217,13 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) { /* Use the XADD MAXLEN 0 trick to generate an empty stream if * the key we are serializing is an empty string, which is possible * for the Stream type. */ + id.ms = 0; id.seq = 1; if (rioWriteBulkCount(r,'*',7) == 0) return 0; if (rioWriteBulkString(r,"XADD",4) == 0) return 0; if (rioWriteBulkObject(r,key) == 0) return 0; if (rioWriteBulkString(r,"MAXLEN",6) == 0) return 0; if (rioWriteBulkString(r,"0",1) == 0) return 0; - if (rioWriteBulkStreamID(r,&s->last_id) == 0) return 0; + if (rioWriteBulkStreamID(r,&id) == 0) return 0; if (rioWriteBulkString(r,"x",1) == 0) return 0; if (rioWriteBulkString(r,"y",1) == 0) return 0; } @@ -1389,9 +1418,11 @@ int rewriteAppendOnlyFile(char *filename) { if (server.aof_rewrite_incremental_fsync) rioSetAutoSync(&aof,REDIS_AUTOSYNC_BYTES); + startSaving(RDBFLAGS_AOF_PREAMBLE); + if (server.aof_use_rdb_preamble) { int error; - if (rdbSaveRio(&aof,&error,RDB_SAVE_AOF_PREAMBLE,NULL) == C_ERR) { + if (rdbSaveRio(&aof,&error,RDBFLAGS_AOF_PREAMBLE,NULL) == C_ERR) { errno = error; goto werr; } @@ -1454,15 +1485,18 @@ int rewriteAppendOnlyFile(char *filename) { if (rename(tmpfile,filename) == -1) { serverLog(LL_WARNING,"Error moving temp append only file on the final destination: %s", strerror(errno)); unlink(tmpfile); + stopSaving(0); return C_ERR; } serverLog(LL_NOTICE,"SYNC append only file rewrite performed"); + stopSaving(1); return C_OK; werr: serverLog(LL_WARNING,"Write error writing append only file on disk: %s", strerror(errno)); fclose(fp); unlink(tmpfile); + stopSaving(0); return C_ERR; } @@ -1558,39 +1592,25 @@ void aofClosePipes(void) { */ int rewriteAppendOnlyFileBackground(void) { pid_t childpid; - long long start; - if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) return C_ERR; + if (hasActiveChildProcess()) return C_ERR; if (aofCreatePipes() != C_OK) return C_ERR; openChildInfoPipe(); - start = ustime(); - if ((childpid = fork()) == 0) { + if ((childpid = redisFork()) == 0) { char tmpfile[256]; /* Child */ - closeListeningSockets(0); redisSetProcTitle("redis-aof-rewrite"); + redisSetCpuAffinity(server.aof_rewrite_cpulist); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid()); if (rewriteAppendOnlyFile(tmpfile) == C_OK) { - size_t private_dirty = zmalloc_get_private_dirty(-1); - - if (private_dirty) { - serverLog(LL_NOTICE, - "AOF rewrite: %zu MB of memory used by copy-on-write", - private_dirty/(1024*1024)); - } - - server.child_info_data.cow_size = private_dirty; - sendChildInfo(CHILD_INFO_TYPE_AOF); + sendChildCOWInfo(CHILD_INFO_TYPE_AOF, "AOF rewrite"); exitFromChild(0); } else { exitFromChild(1); } } else { /* Parent */ - server.stat_fork_time = ustime()-start; - server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ - latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); if (childpid == -1) { closeChildInfoPipe(); serverLog(LL_WARNING, @@ -1604,7 +1624,6 @@ int rewriteAppendOnlyFileBackground(void) { server.aof_rewrite_scheduled = 0; server.aof_rewrite_time_start = time(NULL); server.aof_child_pid = childpid; - updateDictResizePolicy(); /* We set appendseldb to -1 in order to force the next call to the * feedAppendOnlyFile() to issue a SELECT command, so the differences * accumulated by the parent into server.aof_rewrite_buf will start @@ -1619,13 +1638,14 @@ int rewriteAppendOnlyFileBackground(void) { void bgrewriteaofCommand(client *c) { if (server.aof_child_pid != -1) { addReplyError(c,"Background append only file rewriting already in progress"); - } else if (server.rdb_child_pid != -1) { + } else if (hasActiveChildProcess()) { server.aof_rewrite_scheduled = 1; addReplyStatus(c,"Background append only file rewriting scheduled"); } else if (rewriteAppendOnlyFileBackground() == C_OK) { addReplyStatus(c,"Background append only file rewriting started"); } else { - addReply(c,shared.err); + addReplyError(c,"Can't execute an AOF background rewriting. " + "Please check the server logs for more information."); } } @@ -1634,6 +1654,9 @@ void aofRemoveTempFile(pid_t childpid) { snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) childpid); unlink(tmpfile); + + snprintf(tmpfile,256,"temp-rewriteaof-%d.aof", (int) childpid); + unlink(tmpfile); } /* Update the server.aof_current_size field explicitly using stat(2) @@ -1761,7 +1784,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { server.aof_selected_db = -1; /* Make sure SELECT is re-issued */ aofUpdateCurrentSize(); server.aof_rewrite_base_size = server.aof_current_size; - server.aof_current_size = server.aof_current_size; + server.aof_fsync_offset = server.aof_current_size; /* Clear regular AOF buffer since its contents was just written to * the new AOF from the background rewrite buffer. */ @@ -1782,14 +1805,15 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { serverLog(LL_VERBOSE, "Background AOF rewrite signal handler took %lldus", ustime()-now); } else if (!bysignal && exitcode != 0) { + server.aof_lastbgrewrite_status = C_ERR; + + serverLog(LL_WARNING, + "Background AOF rewrite terminated with error"); + } else { /* SIGUSR1 is whitelisted, so we have a way to kill a child without * tirggering an error condition. */ if (bysignal != SIGUSR1) server.aof_lastbgrewrite_status = C_ERR; - serverLog(LL_WARNING, - "Background AOF rewrite terminated with error"); - } else { - server.aof_lastbgrewrite_status = C_ERR; serverLog(LL_WARNING, "Background AOF rewrite terminated by signal %d", bysignal); diff --git a/redis.submodule/src/asciilogo.h b/redis.submodule/src/asciilogo.h index 83c538b..044ca0c 100644 --- a/redis.submodule/src/asciilogo.h +++ b/redis.submodule/src/asciilogo.h @@ -27,7 +27,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ -char *ascii_logo = +const char *ascii_logo = " _._ \n" " _.-``__ ''-._ \n" " _.-`` `. `_. ''-._ Redis %s (%s/%d) %s bit\n" diff --git a/redis.submodule/src/bio.c b/redis.submodule/src/bio.c index 2af6845..69c62fc 100644 --- a/redis.submodule/src/bio.c +++ b/redis.submodule/src/bio.c @@ -154,6 +154,20 @@ void *bioProcessBackgroundJobs(void *arg) { return NULL; } + switch (type) { + case BIO_CLOSE_FILE: + redis_set_thread_title("bio_close_file"); + break; + case BIO_AOF_FSYNC: + redis_set_thread_title("bio_aof_fsync"); + break; + case BIO_LAZY_FREE: + redis_set_thread_title("bio_lazy_free"); + break; + } + + redisSetCpuAffinity(server.bio_cpulist); + /* Make the thread killable at any time, so that bioKillThreads() * can work reliably. */ pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); @@ -254,7 +268,7 @@ void bioKillThreads(void) { int err, j; for (j = 0; j < BIO_NUM_OPS; j++) { - if (pthread_cancel(bio_threads[j]) == 0) { + if (bio_threads[j] && pthread_cancel(bio_threads[j]) == 0) { if ((err = pthread_join(bio_threads[j],NULL)) != 0) { serverLog(LL_WARNING, "Bio thread for job type #%d can be joined: %s", diff --git a/redis.submodule/src/bio.h b/redis.submodule/src/bio.h index 4b15d1c..6c21559 100644 --- a/redis.submodule/src/bio.h +++ b/redis.submodule/src/bio.h @@ -27,6 +27,9 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#ifndef __BIO_H +#define __BIO_H + /* Exported API */ void bioInit(void); void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3); @@ -40,3 +43,5 @@ void bioKillThreads(void); #define BIO_AOF_FSYNC 1 /* Deferred AOF fsync. */ #define BIO_LAZY_FREE 2 /* Deferred objects freeing. */ #define BIO_NUM_OPS 3 + +#endif diff --git a/redis.submodule/src/bitops.c b/redis.submodule/src/bitops.c index 7328d63..f506a88 100644 --- a/redis.submodule/src/bitops.c +++ b/redis.submodule/src/bitops.c @@ -269,7 +269,7 @@ int64_t getSignedBitfield(unsigned char *p, uint64_t offset, uint64_t bits) { * then zero is returned, otherwise in case of overflow, 1 is returned, * otherwise in case of underflow, -1 is returned. * - * When non-zero is returned (oferflow or underflow), if not NULL, *limit is + * When non-zero is returned (overflow or underflow), if not NULL, *limit is * set to the value the operation should result when an overflow happens, * depending on the specified overflow semantics: * @@ -554,7 +554,7 @@ void setbitCommand(client *c) { byteval &= ~(1 << bit); byteval |= ((on & 0x1) << bit); ((uint8_t*)o->ptr)[byte] = byteval; - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id); server.dirty++; addReply(c, bitval ? shared.cone : shared.czero); @@ -754,11 +754,11 @@ void bitopCommand(client *c) { /* Store the computed value into the target key */ if (maxlen) { o = createObject(OBJ_STRING,res); - setKey(c->db,targetkey,o); + setKey(c,c->db,targetkey,o); notifyKeyspaceEvent(NOTIFY_STRING,"set",targetkey,c->db->id); decrRefCount(o); } else if (dbDelete(c->db,targetkey)) { - signalModifiedKey(c->db,targetkey); + signalModifiedKey(c,c->db,targetkey); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",targetkey,c->db->id); } server.dirty++; @@ -902,6 +902,9 @@ void bitposCommand(client *c) { * OVERFLOW [WRAP|SAT|FAIL] */ +#define BITFIELD_FLAG_NONE 0 +#define BITFIELD_FLAG_READONLY (1<<0) + struct bitfieldOp { uint64_t offset; /* Bitfield offset. */ int64_t i64; /* Increment amount (INCRBY) or SET value */ @@ -911,7 +914,10 @@ struct bitfieldOp { int sign; /* True if signed, otherwise unsigned op. */ }; -void bitfieldCommand(client *c) { +/* This implements both the BITFIELD command and the BITFIELD_RO command + * when flags is set to BITFIELD_FLAG_READONLY: in this case only the + * GET subcommand is allowed, other subcommands will return an error. */ +void bitfieldGeneric(client *c, int flags) { robj *o; size_t bitoffset; int j, numops = 0, changes = 0; @@ -999,6 +1005,12 @@ void bitfieldCommand(client *c) { return; } } else { + if (flags & BITFIELD_FLAG_READONLY) { + zfree(ops); + addReplyError(c, "BITFIELD_RO only supports the GET subcommand"); + return; + } + /* Lookup by making room up to the farest bit reached by * this operation. */ if ((o = lookupStringForBitCommand(c, @@ -1008,7 +1020,7 @@ void bitfieldCommand(client *c) { } } - addReplyMultiBulkLen(c,numops); + addReplyArrayLen(c,numops); /* Actually process the operations. */ for (j = 0; j < numops; j++) { @@ -1053,7 +1065,7 @@ void bitfieldCommand(client *c) { setSignedBitfield(o->ptr,thisop->offset, thisop->bits,newval); } else { - addReply(c,shared.nullbulk); + addReplyNull(c); } } else { uint64_t oldval, newval, wrapped, retval; @@ -1082,7 +1094,7 @@ void bitfieldCommand(client *c) { setUnsignedBitfield(o->ptr,thisop->offset, thisop->bits,newval); } else { - addReply(c,shared.nullbulk); + addReplyNull(c); } } changes++; @@ -1123,9 +1135,17 @@ void bitfieldCommand(client *c) { } if (changes) { - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id); server.dirty += changes; } zfree(ops); } + +void bitfieldCommand(client *c) { + bitfieldGeneric(c, BITFIELD_FLAG_NONE); +} + +void bitfieldroCommand(client *c) { + bitfieldGeneric(c, BITFIELD_FLAG_READONLY); +} diff --git a/redis.submodule/src/blocked.c b/redis.submodule/src/blocked.c index 2b43f2b..92f1cee 100644 --- a/redis.submodule/src/blocked.c +++ b/redis.submodule/src/blocked.c @@ -31,9 +31,6 @@ * * API: * - * getTimeoutFromObjectOrReply() is just an utility function to parse a - * timeout argument since blocking operations usually require a timeout. - * * blockClient() set the CLIENT_BLOCKED flag in the client, and set the * specified block type 'btype' filed to one of BLOCKED_* macros. * @@ -67,34 +64,20 @@ int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int where); -/* Get a timeout value from an object and store it into 'timeout'. - * The final timeout is always stored as milliseconds as a time where the - * timeout will expire, however the parsing is performed according to - * the 'unit' that can be seconds or milliseconds. - * - * Note that if the timeout is zero (usually from the point of view of - * commands API this means no timeout) the value stored into 'timeout' - * is zero. */ -int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int unit) { - long long tval; - - if (getLongLongFromObjectOrReply(c,object,&tval, - "timeout is not an integer or out of range") != C_OK) - return C_ERR; - - if (tval < 0) { - addReplyError(c,"timeout is negative"); - return C_ERR; - } - - if (tval > 0) { - if (unit == UNIT_SECONDS) tval *= 1000; - tval += mstime(); - } - *timeout = tval; - - return C_OK; -} +/* This structure represents the blocked key information that we store + * in the client structure. Each client blocked on keys, has a + * client->bpop.keys hash table. The keys of the hash table are Redis + * keys pointers to 'robj' structures. The value is this structure. + * The structure has two goals: firstly we store the list node that this + * client uses to be listed in the database "blocked clients for this key" + * list, so we can later unblock in O(1) without a list scan. + * Secondly for certain blocking types, we have additional info. Right now + * the only use for additional info we have is when clients are blocked + * on streams, as we have to remember the ID it blocked for. */ +typedef struct bkinfo { + listNode *listnode; /* List node for db->blocking_keys[key] list. */ + streamID stream_id; /* Stream ID if we blocked in a stream. */ +} bkinfo; /* Block a client for the specific operation type. Once the CLIENT_BLOCKED * flag is set client query buffer is not longer processed, but accumulated, @@ -104,6 +87,7 @@ void blockClient(client *c, int btype) { c->btype = btype; server.blocked_clients++; server.blocked_clients_by_type[btype]++; + addClientToTimeoutTable(c); } /* This function is called in the beforeSleep() function of the event loop @@ -126,7 +110,7 @@ void processUnblockedClients(void) { * the code is conceptually more correct this way. */ if (!(c->flags & CLIENT_BLOCKED)) { if (c->querybuf && sdslen(c->querybuf) > 0) { - processInputBufferAndReplicate(c); + processInputBuffer(c); } } } @@ -167,6 +151,7 @@ void unblockClient(client *c) { } else if (c->btype == BLOCKED_WAIT) { unblockClientWaitingReplicas(c); } else if (c->btype == BLOCKED_MODULE) { + if (moduleClientIsBlockedOnKeys(c)) unblockClientWaitingData(c); unblockClientFromModule(c); } else { serverPanic("Unknown btype in unblockClient()."); @@ -177,6 +162,7 @@ void unblockClient(client *c) { server.blocked_clients_by_type[c->btype]--; c->flags &= ~CLIENT_BLOCKED; c->btype = BLOCKED_NONE; + removeClientFromTimeoutTable(c); queueClientForReprocessing(c); } @@ -187,7 +173,7 @@ void replyToBlockedClientTimedOut(client *c) { if (c->btype == BLOCKED_LIST || c->btype == BLOCKED_ZSET || c->btype == BLOCKED_STREAM) { - addReply(c,shared.nullmultibulk); + addReplyNullArray(c); } else if (c->btype == BLOCKED_WAIT) { addReplyLongLong(c,replicationCountAcksByOffset(c->bpop.reploffset)); } else if (c->btype == BLOCKED_MODULE) { @@ -222,6 +208,248 @@ void disconnectAllBlockedClients(void) { } } +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * when there may be clients blocked on a list key, and there may be new + * data to fetch (the key is ready). */ +void serveClientsBlockedOnListKey(robj *o, readyList *rl) { + /* We serve clients in the same order they blocked for + * this key, from the first blocked to the last. */ + dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); + if (de) { + list *clients = dictGetVal(de); + int numclients = listLength(clients); + + while(numclients--) { + listNode *clientnode = listFirst(clients); + client *receiver = clientnode->value; + + if (receiver->btype != BLOCKED_LIST) { + /* Put at the tail, so that at the next call + * we'll not run into it again. */ + listRotateHeadToTail(clients); + continue; + } + + robj *dstkey = receiver->bpop.target; + int where = (receiver->lastcmd && + receiver->lastcmd->proc == blpopCommand) ? + LIST_HEAD : LIST_TAIL; + robj *value = listTypePop(o,where); + + if (value) { + /* Protect receiver->bpop.target, that will be + * freed by the next unblockClient() + * call. */ + if (dstkey) incrRefCount(dstkey); + unblockClient(receiver); + + if (serveClientBlockedOnList(receiver, + rl->key,dstkey,rl->db,value, + where) == C_ERR) + { + /* If we failed serving the client we need + * to also undo the POP operation. */ + listTypePush(o,value,where); + } + + if (dstkey) decrRefCount(dstkey); + decrRefCount(value); + } else { + break; + } + } + } + + if (listTypeLength(o) == 0) { + dbDelete(rl->db,rl->key); + notifyKeyspaceEvent(NOTIFY_GENERIC,"del",rl->key,rl->db->id); + } + /* We don't call signalModifiedKey() as it was already called + * when an element was pushed on the list. */ +} + +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * when there may be clients blocked on a sorted set key, and there may be new + * data to fetch (the key is ready). */ +void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) { + /* We serve clients in the same order they blocked for + * this key, from the first blocked to the last. */ + dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); + if (de) { + list *clients = dictGetVal(de); + int numclients = listLength(clients); + unsigned long zcard = zsetLength(o); + + while(numclients-- && zcard) { + listNode *clientnode = listFirst(clients); + client *receiver = clientnode->value; + + if (receiver->btype != BLOCKED_ZSET) { + /* Put at the tail, so that at the next call + * we'll not run into it again. */ + listRotateHeadToTail(clients); + continue; + } + + int where = (receiver->lastcmd && + receiver->lastcmd->proc == bzpopminCommand) + ? ZSET_MIN : ZSET_MAX; + unblockClient(receiver); + genericZpopCommand(receiver,&rl->key,1,where,1,NULL); + zcard--; + + /* Replicate the command. */ + robj *argv[2]; + struct redisCommand *cmd = where == ZSET_MIN ? + server.zpopminCommand : + server.zpopmaxCommand; + argv[0] = createStringObject(cmd->name,strlen(cmd->name)); + argv[1] = rl->key; + incrRefCount(rl->key); + propagate(cmd,receiver->db->id, + argv,2,PROPAGATE_AOF|PROPAGATE_REPL); + decrRefCount(argv[0]); + decrRefCount(argv[1]); + } + } +} + +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * when there may be clients blocked on a stream key, and there may be new + * data to fetch (the key is ready). */ +void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { + dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); + stream *s = o->ptr; + + /* We need to provide the new data arrived on the stream + * to all the clients that are waiting for an offset smaller + * than the current top item. */ + if (de) { + list *clients = dictGetVal(de); + listNode *ln; + listIter li; + listRewind(clients,&li); + + while((ln = listNext(&li))) { + client *receiver = listNodeValue(ln); + if (receiver->btype != BLOCKED_STREAM) continue; + bkinfo *bki = dictFetchValue(receiver->bpop.keys,rl->key); + streamID *gt = &bki->stream_id; + + /* If we blocked in the context of a consumer + * group, we need to resolve the group and update the + * last ID the client is blocked for: this is needed + * because serving other clients in the same consumer + * group will alter the "last ID" of the consumer + * group, and clients blocked in a consumer group are + * always blocked for the ">" ID: we need to deliver + * only new messages and avoid unblocking the client + * otherwise. */ + streamCG *group = NULL; + if (receiver->bpop.xread_group) { + group = streamLookupCG(s, + receiver->bpop.xread_group->ptr); + /* If the group was not found, send an error + * to the consumer. */ + if (!group) { + addReplyError(receiver, + "-NOGROUP the consumer group this client " + "was blocked on no longer exists"); + unblockClient(receiver); + continue; + } else { + *gt = group->last_id; + } + } + + if (streamCompareID(&s->last_id, gt) > 0) { + streamID start = *gt; + streamIncrID(&start); + + /* Lookup the consumer for the group, if any. */ + streamConsumer *consumer = NULL; + int noack = 0; + + if (group) { + consumer = + streamLookupConsumer(group, + receiver->bpop.xread_consumer->ptr, + SLC_NONE); + noack = receiver->bpop.xread_group_noack; + } + + /* Emit the two elements sub-array consisting of + * the name of the stream and the data we + * extracted from it. Wrapped in a single-item + * array, since we have just one key. */ + if (receiver->resp == 2) { + addReplyArrayLen(receiver,1); + addReplyArrayLen(receiver,2); + } else { + addReplyMapLen(receiver,1); + } + addReplyBulk(receiver,rl->key); + + streamPropInfo pi = { + rl->key, + receiver->bpop.xread_group + }; + streamReplyWithRange(receiver,s,&start,NULL, + receiver->bpop.xread_count, + 0, group, consumer, noack, &pi); + + /* Note that after we unblock the client, 'gt' + * and other receiver->bpop stuff are no longer + * valid, so we must do the setup above before + * this call. */ + unblockClient(receiver); + } + } + } +} + +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * in order to check if we can serve clients blocked by modules using + * RM_BlockClientOnKeys(), when the corresponding key was signaled as ready: + * our goal here is to call the RedisModuleBlockedClient reply() callback to + * see if the key is really able to serve the client, and in that case, + * unblock it. */ +void serveClientsBlockedOnKeyByModule(readyList *rl) { + dictEntry *de; + + /* We serve clients in the same order they blocked for + * this key, from the first blocked to the last. */ + de = dictFind(rl->db->blocking_keys,rl->key); + if (de) { + list *clients = dictGetVal(de); + int numclients = listLength(clients); + + while(numclients--) { + listNode *clientnode = listFirst(clients); + client *receiver = clientnode->value; + + /* Put at the tail, so that at the next call + * we'll not run into it again: clients here may not be + * ready to be served, so they'll remain in the list + * sometimes. We want also be able to skip clients that are + * not blocked for the MODULE type safely. */ + listRotateHeadToTail(clients); + + if (receiver->btype != BLOCKED_MODULE) continue; + + /* Note that if *this* client cannot be served by this key, + * it does not mean that another client that is next into the + * list cannot be served as well: they may be blocked by + * different modules with different triggers to consider if a key + * is ready or not. This means we can't exit the loop but need + * to continue after the first failure. */ + if (!moduleTryServeClientBlockedOnKey(receiver, rl->key)) continue; + + moduleUnblockClient(receiver); + } + } +} + /* This function should be called by Redis every time a single command, * a MULTI/EXEC block, or a Lua script, terminated its execution after * being called by a client. It handles serving clients blocked in @@ -262,201 +490,32 @@ void handleClientsBlockedOnKeys(void) { * we can safely call signalKeyAsReady() against this key. */ dictDelete(rl->db->ready_keys,rl->key); + /* Even if we are not inside call(), increment the call depth + * in order to make sure that keys are expired against a fixed + * reference time, and not against the wallclock time. This + * way we can lookup an object multiple times (BRPOPLPUSH does + * that) without the risk of it being freed in the second + * lookup, invalidating the first one. + * See https://github.com/antirez/redis/pull/6554. */ + server.fixed_time_expire++; + updateCachedTime(0); + /* Serve clients blocked on list key. */ robj *o = lookupKeyWrite(rl->db,rl->key); - if (o != NULL && o->type == OBJ_LIST) { - dictEntry *de; - - /* We serve clients in the same order they blocked for - * this key, from the first blocked to the last. */ - de = dictFind(rl->db->blocking_keys,rl->key); - if (de) { - list *clients = dictGetVal(de); - int numclients = listLength(clients); - - while(numclients--) { - listNode *clientnode = listFirst(clients); - client *receiver = clientnode->value; - - if (receiver->btype != BLOCKED_LIST) { - /* Put at the tail, so that at the next call - * we'll not run into it again. */ - listDelNode(clients,clientnode); - listAddNodeTail(clients,receiver); - continue; - } - - robj *dstkey = receiver->bpop.target; - int where = (receiver->lastcmd && - receiver->lastcmd->proc == blpopCommand) ? - LIST_HEAD : LIST_TAIL; - robj *value = listTypePop(o,where); - - if (value) { - /* Protect receiver->bpop.target, that will be - * freed by the next unblockClient() - * call. */ - if (dstkey) incrRefCount(dstkey); - unblockClient(receiver); - - if (serveClientBlockedOnList(receiver, - rl->key,dstkey,rl->db,value, - where) == C_ERR) - { - /* If we failed serving the client we need - * to also undo the POP operation. */ - listTypePush(o,value,where); - } - - if (dstkey) decrRefCount(dstkey); - decrRefCount(value); - } else { - break; - } - } - } - if (listTypeLength(o) == 0) { - dbDelete(rl->db,rl->key); - notifyKeyspaceEvent(NOTIFY_GENERIC,"del",rl->key,rl->db->id); - } - /* We don't call signalModifiedKey() as it was already called - * when an element was pushed on the list. */ - } - - /* Serve clients blocked on sorted set key. */ - else if (o != NULL && o->type == OBJ_ZSET) { - dictEntry *de; - - /* We serve clients in the same order they blocked for - * this key, from the first blocked to the last. */ - de = dictFind(rl->db->blocking_keys,rl->key); - if (de) { - list *clients = dictGetVal(de); - int numclients = listLength(clients); - unsigned long zcard = zsetLength(o); - - while(numclients-- && zcard) { - listNode *clientnode = listFirst(clients); - client *receiver = clientnode->value; - - if (receiver->btype != BLOCKED_ZSET) { - /* Put at the tail, so that at the next call - * we'll not run into it again. */ - listDelNode(clients,clientnode); - listAddNodeTail(clients,receiver); - continue; - } - - int where = (receiver->lastcmd && - receiver->lastcmd->proc == bzpopminCommand) - ? ZSET_MIN : ZSET_MAX; - unblockClient(receiver); - genericZpopCommand(receiver,&rl->key,1,where,1,NULL); - zcard--; - - /* Replicate the command. */ - robj *argv[2]; - struct redisCommand *cmd = where == ZSET_MIN ? - server.zpopminCommand : - server.zpopmaxCommand; - argv[0] = createStringObject(cmd->name,strlen(cmd->name)); - argv[1] = rl->key; - incrRefCount(rl->key); - propagate(cmd,receiver->db->id, - argv,2,PROPAGATE_AOF|PROPAGATE_REPL); - decrRefCount(argv[0]); - decrRefCount(argv[1]); - } - } - } - - /* Serve clients blocked on stream key. */ - else if (o != NULL && o->type == OBJ_STREAM) { - dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); - stream *s = o->ptr; - - /* We need to provide the new data arrived on the stream - * to all the clients that are waiting for an offset smaller - * than the current top item. */ - if (de) { - list *clients = dictGetVal(de); - listNode *ln; - listIter li; - listRewind(clients,&li); - - while((ln = listNext(&li))) { - client *receiver = listNodeValue(ln); - if (receiver->btype != BLOCKED_STREAM) continue; - streamID *gt = dictFetchValue(receiver->bpop.keys, - rl->key); - - /* If we blocked in the context of a consumer - * group, we need to resolve the group and update the - * last ID the client is blocked for: this is needed - * because serving other clients in the same consumer - * group will alter the "last ID" of the consumer - * group, and clients blocked in a consumer group are - * always blocked for the ">" ID: we need to deliver - * only new messages and avoid unblocking the client - * otherwise. */ - streamCG *group = NULL; - if (receiver->bpop.xread_group) { - group = streamLookupCG(s, - receiver->bpop.xread_group->ptr); - /* If the group was not found, send an error - * to the consumer. */ - if (!group) { - addReplyError(receiver, - "-NOGROUP the consumer group this client " - "was blocked on no longer exists"); - unblockClient(receiver); - continue; - } else { - *gt = group->last_id; - } - } - - if (streamCompareID(&s->last_id, gt) > 0) { - streamID start = *gt; - start.seq++; /* Can't overflow, it's an uint64_t */ - - /* Lookup the consumer for the group, if any. */ - streamConsumer *consumer = NULL; - int noack = 0; - - if (group) { - consumer = streamLookupConsumer(group, - receiver->bpop.xread_consumer->ptr, - 1); - noack = receiver->bpop.xread_group_noack; - } - - /* Emit the two elements sub-array consisting of - * the name of the stream and the data we - * extracted from it. Wrapped in a single-item - * array, since we have just one key. */ - addReplyMultiBulkLen(receiver,1); - addReplyMultiBulkLen(receiver,2); - addReplyBulk(receiver,rl->key); - - streamPropInfo pi = { - rl->key, - receiver->bpop.xread_group - }; - streamReplyWithRange(receiver,s,&start,NULL, - receiver->bpop.xread_count, - 0, group, consumer, noack, &pi); - - /* Note that after we unblock the client, 'gt' - * and other receiver->bpop stuff are no longer - * valid, so we must do the setup above before - * this call. */ - unblockClient(receiver); - } - } - } + if (o != NULL) { + if (o->type == OBJ_LIST) + serveClientsBlockedOnListKey(o,rl); + else if (o->type == OBJ_ZSET) + serveClientsBlockedOnSortedSetKey(o,rl); + else if (o->type == OBJ_STREAM) + serveClientsBlockedOnStreamKey(o,rl); + /* We want to serve clients blocked on module keys + * regardless of the object type: we don't know what the + * module is trying to accomplish right now. */ + serveClientsBlockedOnKeyByModule(rl); } + server.fixed_time_expire--; /* Free this item. */ decrRefCount(rl->key); @@ -505,17 +564,15 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo if (target != NULL) incrRefCount(target); for (j = 0; j < numkeys; j++) { - /* The value associated with the key name in the bpop.keys dictionary - * is NULL for lists and sorted sets, or the stream ID for streams. */ - void *key_data = NULL; - if (btype == BLOCKED_STREAM) { - key_data = zmalloc(sizeof(streamID)); - memcpy(key_data,ids+j,sizeof(streamID)); - } + /* Allocate our bkinfo structure, associated to each key the client + * is blocked for. */ + bkinfo *bki = zmalloc(sizeof(*bki)); + if (btype == BLOCKED_STREAM) + bki->stream_id = ids[j]; /* If the key already exists in the dictionary ignore it. */ - if (dictAdd(c->bpop.keys,keys[j],key_data) != DICT_OK) { - zfree(key_data); + if (dictAdd(c->bpop.keys,keys[j],bki) != DICT_OK) { + zfree(bki); continue; } incrRefCount(keys[j]); @@ -534,6 +591,7 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo l = dictGetVal(de); } listAddNodeTail(l,c); + bki->listnode = listLast(l); } blockClient(c,btype); } @@ -550,11 +608,12 @@ void unblockClientWaitingData(client *c) { /* The client may wait for multiple keys, so unblock it for every key. */ while((de = dictNext(di)) != NULL) { robj *key = dictGetKey(de); + bkinfo *bki = dictGetVal(de); /* Remove this client from the list of clients waiting for this key. */ l = dictFetchValue(c->db->blocking_keys,key); serverAssertWithInfo(c,key,l != NULL); - listDelNode(l,listSearchKey(l,c)); + listDelNode(l,bki->listnode); /* If the list is empty we need to remove it to avoid wasting memory */ if (listLength(l) == 0) dictDelete(c->db->blocking_keys,key); @@ -581,7 +640,7 @@ void unblockClientWaitingData(client *c) { * the same key again and again in the list in case of multiple pushes * made by a script or in the context of MULTI/EXEC. * - * The list will be finally processed by handleClientsBlockedOnLists() */ + * The list will be finally processed by handleClientsBlockedOnKeys() */ void signalKeyAsReady(redisDb *db, robj *key) { readyList *rl; diff --git a/redis.submodule/src/childinfo.c b/redis.submodule/src/childinfo.c index 719025e..fa06005 100644 --- a/redis.submodule/src/childinfo.c +++ b/redis.submodule/src/childinfo.c @@ -80,6 +80,8 @@ void receiveChildInfo(void) { server.stat_rdb_cow_bytes = server.child_info_data.cow_size; } else if (server.child_info_data.process_type == CHILD_INFO_TYPE_AOF) { server.stat_aof_cow_bytes = server.child_info_data.cow_size; + } else if (server.child_info_data.process_type == CHILD_INFO_TYPE_MODULE) { + server.stat_module_cow_bytes = server.child_info_data.cow_size; } } } diff --git a/redis.submodule/src/cluster.c b/redis.submodule/src/cluster.c index 858c0c9..24b14d1 100644 --- a/redis.submodule/src/cluster.c +++ b/redis.submodule/src/cluster.c @@ -49,7 +49,7 @@ clusterNode *myself = NULL; clusterNode *createClusterNode(char *nodename, int flags); int clusterAddNode(clusterNode *node); void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask); -void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask); +void clusterReadHandler(connection *conn); void clusterSendPing(clusterLink *link, int type); void clusterSendFail(char *nodename); void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request); @@ -157,7 +157,10 @@ int clusterLoadConfig(char *filename) { } /* Regular config lines have at least eight fields */ - if (argc < 8) goto fmterr; + if (argc < 8) { + sdsfreesplitres(argv,argc); + goto fmterr; + } /* Create this node if it does not exist */ n = clusterLookupNode(argv[0]); @@ -166,7 +169,10 @@ int clusterLoadConfig(char *filename) { clusterAddNode(n); } /* Address and port */ - if ((p = strrchr(argv[1],':')) == NULL) goto fmterr; + if ((p = strrchr(argv[1],':')) == NULL) { + sdsfreesplitres(argv,argc); + goto fmterr; + } *p = '\0'; memcpy(n->ip,argv[1],strlen(argv[1])+1); char *port = p+1; @@ -247,7 +253,10 @@ int clusterLoadConfig(char *filename) { *p = '\0'; direction = p[1]; /* Either '>' or '<' */ slot = atoi(argv[j]+1); - if (slot < 0 || slot >= CLUSTER_SLOTS) goto fmterr; + if (slot < 0 || slot >= CLUSTER_SLOTS) { + sdsfreesplitres(argv,argc); + goto fmterr; + } p += 3; cn = clusterLookupNode(p); if (!cn) { @@ -267,8 +276,12 @@ int clusterLoadConfig(char *filename) { } else { start = stop = atoi(argv[j]); } - if (start < 0 || start >= CLUSTER_SLOTS) goto fmterr; - if (stop < 0 || stop >= CLUSTER_SLOTS) goto fmterr; + if (start < 0 || start >= CLUSTER_SLOTS || + stop < 0 || stop >= CLUSTER_SLOTS) + { + sdsfreesplitres(argv,argc); + goto fmterr; + } while(start <= stop) clusterAddSlot(n, start++); } @@ -477,7 +490,8 @@ void clusterInit(void) { /* Port sanity check II * The other handshake port check is triggered too late to stop * us from trying to use a too-high cluster port number. */ - if (server.port > (65535-CLUSTER_PORT_INCR)) { + int port = server.tls_cluster ? server.tls_port : server.port; + if (port > (65535-CLUSTER_PORT_INCR)) { serverLog(LL_WARNING, "Redis port number too high. " "Cluster communication port is 10,000 port " "numbers higher than your Redis port. " @@ -485,8 +499,7 @@ void clusterInit(void) { "lower than 55535."); exit(1); } - - if (listenToPort(server.port+CLUSTER_PORT_INCR, + if (listenToPort(port+CLUSTER_PORT_INCR, server.cfd,&server.cfd_count) == C_ERR) { exit(1); @@ -508,8 +521,8 @@ void clusterInit(void) { /* Set myself->port / cport to my listening ports, we'll just need to * discover the IP address via MEET messages. */ - myself->port = server.port; - myself->cport = server.port+CLUSTER_PORT_INCR; + myself->port = port; + myself->cport = port+CLUSTER_PORT_INCR; if (server.cluster_announce_port) myself->port = server.cluster_announce_port; if (server.cluster_announce_bus_port) @@ -593,7 +606,7 @@ clusterLink *createClusterLink(clusterNode *node) { link->sndbuf = sdsempty(); link->rcvbuf = sdsempty(); link->node = node; - link->fd = -1; + link->conn = NULL; return link; } @@ -601,23 +614,45 @@ clusterLink *createClusterLink(clusterNode *node) { * This function will just make sure that the original node associated * with this link will have the 'link' field set to NULL. */ void freeClusterLink(clusterLink *link) { - if (link->fd != -1) { - aeDeleteFileEvent(server.el, link->fd, AE_READABLE|AE_WRITABLE); + if (link->conn) { + connClose(link->conn); + link->conn = NULL; } sdsfree(link->sndbuf); sdsfree(link->rcvbuf); if (link->node) link->node->link = NULL; - close(link->fd); zfree(link); } +static void clusterConnAcceptHandler(connection *conn) { + clusterLink *link; + + if (connGetState(conn) != CONN_STATE_CONNECTED) { + serverLog(LL_VERBOSE, + "Error accepting cluster node connection: %s", connGetLastError(conn)); + connClose(conn); + return; + } + + /* Create a link object we use to handle the connection. + * It gets passed to the readable handler when data is available. + * Initiallly the link->node pointer is set to NULL as we don't know + * which node is, but the right node is references once we know the + * node identity. */ + link = createClusterLink(NULL); + link->conn = conn; + connSetPrivateData(conn, link); + + /* Register read handler */ + connSetReadHandler(conn, clusterReadHandler); +} + #define MAX_CLUSTER_ACCEPTS_PER_CALL 1000 void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { int cport, cfd; int max = MAX_CLUSTER_ACCEPTS_PER_CALL; char cip[NET_IP_STR_LEN]; - clusterLink *link; UNUSED(el); UNUSED(mask); UNUSED(privdata); @@ -634,19 +669,25 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { "Error accepting cluster node: %s", server.neterr); return; } - anetNonBlock(NULL,cfd); - anetEnableTcpNoDelay(NULL,cfd); + + connection *conn = server.tls_cluster ? connCreateAcceptedTLS(cfd,1) : connCreateAcceptedSocket(cfd); + connNonBlock(conn); + connEnableTcpNoDelay(conn); /* Use non-blocking I/O for cluster messages. */ - serverLog(LL_VERBOSE,"Accepted cluster node %s:%d", cip, cport); - /* Create a link object we use to handle the connection. - * It gets passed to the readable handler when data is available. - * Initiallly the link->node pointer is set to NULL as we don't know - * which node is, but the right node is references once we know the - * node identity. */ - link = createClusterLink(NULL); - link->fd = cfd; - aeCreateFileEvent(server.el,cfd,AE_READABLE,clusterReadHandler,link); + serverLog(LL_VERBOSE,"Accepting cluster node connection from %s:%d", cip, cport); + + /* Accept the connection now. connAccept() may call our handler directly + * or schedule it for later depending on connection implementation. + */ + if (connAccept(conn, clusterConnAcceptHandler) == C_ERR) { + if (connGetState(conn) == CONN_STATE_ERROR) + serverLog(LL_VERBOSE, + "Error accepting cluster node connection: %s", + connGetLastError(conn)); + connClose(conn); + return; + } } } @@ -708,6 +749,7 @@ clusterNode *createClusterNode(char *nodename, int flags) { node->slaves = NULL; node->slaveof = NULL; node->ping_sent = node->pong_received = 0; + node->data_received = 0; node->fail_time = 0; node->link = NULL; memset(node->ip,0,sizeof(node->ip)); @@ -892,7 +934,7 @@ int clusterAddNode(clusterNode *node) { return (retval == DICT_OK) ? C_OK : C_ERR; } -/* Remove a node from the cluster. The functio performs the high level +/* Remove a node from the cluster. The function performs the high level * cleanup, calling freeClusterNode() for the low level cleanup. * Here we do the following: * @@ -1421,7 +1463,10 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) { } } else { /* If it's not in NOADDR state and we don't have it, we - * start a handshake process against this IP/PORT pairs. + * add it to our trusted dict with exact nodeid and flag. + * Note that we cannot simply start a handshake against + * this IP/PORT pairs, since IP/PORT can be reused already, + * otherwise we risk joining another cluster. * * Note that we require that the sender of this gossip message * is a well known node in our cluster, otherwise we risk @@ -1430,7 +1475,12 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) { !(flags & CLUSTER_NODE_NOADDR) && !clusterBlacklistExists(g->nodename)) { - clusterStartHandshake(g->ip,ntohs(g->port),ntohs(g->cport)); + clusterNode *node; + node = createClusterNode(g->nodename, flags); + memcpy(node->ip,g->ip,NET_IP_STR_LEN); + node->port = ntohs(g->port); + node->cport = ntohs(g->cport); + clusterAddNode(node); } } @@ -1447,7 +1497,7 @@ void nodeIp2String(char *buf, clusterLink *link, char *announced_ip) { memcpy(buf,announced_ip,NET_IP_STR_LEN); buf[NET_IP_STR_LEN-1] = '\0'; /* We are not sure the input is sane. */ } else { - anetPeerToString(link->fd, buf, NET_IP_STR_LEN, NULL); + connPeerToString(link->conn, buf, NET_IP_STR_LEN, NULL); } } @@ -1637,6 +1687,7 @@ int clusterProcessPacket(clusterLink *link) { clusterMsg *hdr = (clusterMsg*) link->rcvbuf; uint32_t totlen = ntohl(hdr->totlen); uint16_t type = ntohs(hdr->type); + mstime_t now = mstime(); if (type < CLUSTERMSG_TYPE_COUNT) server.cluster->stats_bus_messages_received[type]++; @@ -1698,8 +1749,17 @@ int clusterProcessPacket(clusterLink *link) { if (totlen != explen) return 1; } - /* Check if the sender is a known node. */ + /* Check if the sender is a known node. Note that for incoming connections + * we don't store link->node information, but resolve the node by the + * ID in the header each time in the current implementation. */ sender = clusterLookupNode(hdr->sender); + + /* Update the last time we saw any data from this node. We + * use this in order to avoid detecting a timeout from a node that + * is just sending a lot of data in the cluster bus, for instance + * because of Pub/Sub. */ + if (sender) sender->data_received = now; + if (sender && !nodeInHandshake(sender)) { /* Update our curretEpoch if we see a newer epoch in the cluster. */ senderCurrentEpoch = ntohu64(hdr->currentEpoch); @@ -1714,7 +1774,7 @@ int clusterProcessPacket(clusterLink *link) { } /* Update the replication offset info for this node. */ sender->repl_offset = ntohu64(hdr->offset); - sender->repl_offset_time = mstime(); + sender->repl_offset_time = now; /* If we are a slave performing a manual failover and our master * sent its offset while already paused, populate the MF state. */ if (server.cluster->mf_end && @@ -1751,7 +1811,7 @@ int clusterProcessPacket(clusterLink *link) { { char ip[NET_IP_STR_LEN]; - if (anetSockName(link->fd,ip,sizeof(ip),NULL) != -1 && + if (connSockName(link->conn,ip,sizeof(ip),NULL) != -1 && strcmp(ip,myself->ip)) { memcpy(myself->ip,ip,NET_IP_STR_LEN); @@ -1828,7 +1888,7 @@ int clusterProcessPacket(clusterLink *link) { * address. */ serverLog(LL_DEBUG,"PONG contains mismatching sender ID. About node %.40s added %d ms ago, having flags %d", link->node->name, - (int)(mstime()-(link->node->ctime)), + (int)(now-(link->node->ctime)), link->node->flags); link->node->flags |= CLUSTER_NODE_NOADDR; link->node->ip[0] = '\0'; @@ -1863,7 +1923,7 @@ int clusterProcessPacket(clusterLink *link) { /* Update our info about the node */ if (link->node && type == CLUSTERMSG_TYPE_PONG) { - link->node->pong_received = mstime(); + link->node->pong_received = now; link->node->ping_sent = 0; /* The PFAIL condition can be reversed without external @@ -2010,7 +2070,7 @@ int clusterProcessPacket(clusterLink *link) { "FAIL message received from %.40s about %.40s", hdr->sender, hdr->data.fail.about.nodename); failing->flags |= CLUSTER_NODE_FAIL; - failing->fail_time = mstime(); + failing->fail_time = now; failing->flags &= ~CLUSTER_NODE_PFAIL; clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG| CLUSTER_TODO_UPDATE_STATE); @@ -2063,9 +2123,9 @@ int clusterProcessPacket(clusterLink *link) { /* Manual failover requested from slaves. Initialize the state * accordingly. */ resetManualFailover(); - server.cluster->mf_end = mstime() + CLUSTER_MF_TIMEOUT; + server.cluster->mf_end = now + CLUSTER_MF_TIMEOUT; server.cluster->mf_slave = sender; - pauseClients(mstime()+(CLUSTER_MF_TIMEOUT*2)); + pauseClients(now+(CLUSTER_MF_TIMEOUT*CLUSTER_MF_PAUSE_MULT)); serverLog(LL_WARNING,"Manual failover requested by replica %.40s.", sender->name); } else if (type == CLUSTERMSG_TYPE_UPDATE) { @@ -2118,35 +2178,76 @@ void handleLinkIOError(clusterLink *link) { /* Send data. This is handled using a trivial send buffer that gets * consumed by write(). We don't try to optimize this for speed too much * as this is a very low traffic channel. */ -void clusterWriteHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - clusterLink *link = (clusterLink*) privdata; +void clusterWriteHandler(connection *conn) { + clusterLink *link = connGetPrivateData(conn); ssize_t nwritten; - UNUSED(el); - UNUSED(mask); - nwritten = write(fd, link->sndbuf, sdslen(link->sndbuf)); + nwritten = connWrite(conn, link->sndbuf, sdslen(link->sndbuf)); if (nwritten <= 0) { serverLog(LL_DEBUG,"I/O error writing to node link: %s", - (nwritten == -1) ? strerror(errno) : "short write"); + (nwritten == -1) ? connGetLastError(conn) : "short write"); handleLinkIOError(link); return; } sdsrange(link->sndbuf,nwritten,-1); if (sdslen(link->sndbuf) == 0) - aeDeleteFileEvent(server.el, link->fd, AE_WRITABLE); + connSetWriteHandler(link->conn, NULL); +} + +/* A connect handler that gets called when a connection to another node + * gets established. + */ +void clusterLinkConnectHandler(connection *conn) { + clusterLink *link = connGetPrivateData(conn); + clusterNode *node = link->node; + + /* Check if connection succeeded */ + if (connGetState(conn) != CONN_STATE_CONNECTED) { + serverLog(LL_VERBOSE, "Connection with Node %.40s at %s:%d failed: %s", + node->name, node->ip, node->cport, + connGetLastError(conn)); + freeClusterLink(link); + return; + } + + /* Register a read handler from now on */ + connSetReadHandler(conn, clusterReadHandler); + + /* Queue a PING in the new connection ASAP: this is crucial + * to avoid false positives in failure detection. + * + * If the node is flagged as MEET, we send a MEET message instead + * of a PING one, to force the receiver to add us in its node + * table. */ + mstime_t old_ping_sent = node->ping_sent; + clusterSendPing(link, node->flags & CLUSTER_NODE_MEET ? + CLUSTERMSG_TYPE_MEET : CLUSTERMSG_TYPE_PING); + if (old_ping_sent) { + /* If there was an active ping before the link was + * disconnected, we want to restore the ping time, otherwise + * replaced by the clusterSendPing() call. */ + node->ping_sent = old_ping_sent; + } + /* We can clear the flag after the first packet is sent. + * If we'll never receive a PONG, we'll never send new packets + * to this node. Instead after the PONG is received and we + * are no longer in meet/handshake status, we want to send + * normal PING packets. */ + node->flags &= ~CLUSTER_NODE_MEET; + + serverLog(LL_DEBUG,"Connecting with Node %.40s at %s:%d", + node->name, node->ip, node->cport); } /* Read data. Try to read the first field of the header first to check the * full length of the packet. When a whole packet is in memory this function * will call the function to process the packet. And so forth. */ -void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - char buf[sizeof(clusterMsg)]; +void clusterReadHandler(connection *conn) { + clusterMsg buf[1]; ssize_t nread; clusterMsg *hdr; - clusterLink *link = (clusterLink*) privdata; + clusterLink *link = connGetPrivateData(conn); unsigned int readlen, rcvbuflen; - UNUSED(el); - UNUSED(mask); while(1) { /* Read as long as there is data to read. */ rcvbuflen = sdslen(link->rcvbuf); @@ -2174,13 +2275,13 @@ void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { if (readlen > sizeof(buf)) readlen = sizeof(buf); } - nread = read(fd,buf,readlen); - if (nread == -1 && errno == EAGAIN) return; /* No more data ready. */ + nread = connRead(conn,buf,readlen); + if (nread == -1 && (connGetState(conn) == CONN_STATE_CONNECTED)) return; /* No more data ready. */ if (nread <= 0) { /* I/O error... */ serverLog(LL_DEBUG,"I/O error reading from node link: %s", - (nread == 0) ? "connection closed" : strerror(errno)); + (nread == 0) ? "connection closed" : connGetLastError(conn)); handleLinkIOError(link); return; } else { @@ -2209,8 +2310,7 @@ void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { * from event handlers that will do stuff with the same link later. */ void clusterSendMessage(clusterLink *link, unsigned char *msg, size_t msglen) { if (sdslen(link->sndbuf) == 0 && msglen != 0) - aeCreateFileEvent(server.el,link->fd,AE_WRITABLE|AE_BARRIER, - clusterWriteHandler,link); + connSetWriteHandlerWithBarrier(link->conn, clusterWriteHandler, 1); link->sndbuf = sdscatlen(link->sndbuf, msg, msglen); @@ -2276,11 +2376,12 @@ void clusterBuildMessageHdr(clusterMsg *hdr, int type) { } /* Handle cluster-announce-port as well. */ + int port = server.tls_cluster ? server.tls_port : server.port; int announced_port = server.cluster_announce_port ? - server.cluster_announce_port : server.port; + server.cluster_announce_port : port; int announced_cport = server.cluster_announce_bus_port ? server.cluster_announce_bus_port : - (server.port + CLUSTER_PORT_INCR); + (port + CLUSTER_PORT_INCR); memcpy(hdr->myslots,master->slots,sizeof(hdr->myslots)); memset(hdr->slaveof,0,CLUSTER_NAMELEN); @@ -2517,7 +2618,8 @@ void clusterBroadcastPong(int target) { * * If link is NULL, then the message is broadcasted to the whole cluster. */ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { - unsigned char buf[sizeof(clusterMsg)], *payload; + unsigned char *payload; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; uint32_t channel_len, message_len; @@ -2537,7 +2639,7 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { /* Try to use the local buffer if possible */ if (totlen < sizeof(buf)) { - payload = buf; + payload = (unsigned char*)buf; } else { payload = zmalloc(totlen); memcpy(payload,hdr,sizeof(*hdr)); @@ -2554,7 +2656,7 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { decrRefCount(channel); decrRefCount(message); - if (payload != buf) zfree(payload); + if (payload != (unsigned char*)buf) zfree(payload); } /* Send a FAIL message to all the nodes we are able to contact. @@ -2563,7 +2665,7 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { * we switch the node state to CLUSTER_NODE_FAIL and ask all the other * nodes to do the same ASAP. */ void clusterSendFail(char *nodename) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_FAIL); @@ -2575,7 +2677,7 @@ void clusterSendFail(char *nodename) { * slots configuration. The node name, slots bitmap, and configEpoch info * are included. */ void clusterSendUpdate(clusterLink *link, clusterNode *node) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; if (link == NULL) return; @@ -2583,7 +2685,7 @@ void clusterSendUpdate(clusterLink *link, clusterNode *node) { memcpy(hdr->data.update.nodecfg.nodename,node->name,CLUSTER_NAMELEN); hdr->data.update.nodecfg.configEpoch = htonu64(node->configEpoch); memcpy(hdr->data.update.nodecfg.slots,node->slots,sizeof(node->slots)); - clusterSendMessage(link,buf,ntohl(hdr->totlen)); + clusterSendMessage(link,(unsigned char*)buf,ntohl(hdr->totlen)); } /* Send a MODULE message. @@ -2591,7 +2693,8 @@ void clusterSendUpdate(clusterLink *link, clusterNode *node) { * If link is NULL, then the message is broadcasted to the whole cluster. */ void clusterSendModule(clusterLink *link, uint64_t module_id, uint8_t type, unsigned char *payload, uint32_t len) { - unsigned char buf[sizeof(clusterMsg)], *heapbuf; + unsigned char *heapbuf; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2606,7 +2709,7 @@ void clusterSendModule(clusterLink *link, uint64_t module_id, uint8_t type, /* Try to use the local buffer if possible */ if (totlen < sizeof(buf)) { - heapbuf = buf; + heapbuf = (unsigned char*)buf; } else { heapbuf = zmalloc(totlen); memcpy(heapbuf,hdr,sizeof(*hdr)); @@ -2619,7 +2722,7 @@ void clusterSendModule(clusterLink *link, uint64_t module_id, uint8_t type, else clusterBroadcastMessage(heapbuf,totlen); - if (heapbuf != buf) zfree(heapbuf); + if (heapbuf != (unsigned char*)buf) zfree(heapbuf); } /* This function gets a cluster node ID string as target, the same way the nodes @@ -2663,7 +2766,7 @@ void clusterPropagatePublish(robj *channel, robj *message) { * Note that we send the failover request to everybody, master and slave nodes, * but only the masters are supposed to reply to our query. */ void clusterRequestFailoverAuth(void) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2679,7 +2782,7 @@ void clusterRequestFailoverAuth(void) { /* Send a FAILOVER_AUTH_ACK message to the specified node. */ void clusterSendFailoverAuth(clusterNode *node) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2687,12 +2790,12 @@ void clusterSendFailoverAuth(clusterNode *node) { clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK); totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); hdr->totlen = htonl(totlen); - clusterSendMessage(node->link,buf,totlen); + clusterSendMessage(node->link,(unsigned char*)buf,totlen); } /* Send a MFSTART message to the specified node. */ void clusterSendMFStart(clusterNode *node) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2700,7 +2803,7 @@ void clusterSendMFStart(clusterNode *node) { clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_MFSTART); totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); hdr->totlen = htonl(totlen); - clusterSendMessage(node->link,buf,totlen); + clusterSendMessage(node->link,(unsigned char*)buf,totlen); } /* Vote for the node asking for our vote if there are the conditions. */ @@ -3383,13 +3486,11 @@ void clusterCron(void) { } if (node->link == NULL) { - int fd; - mstime_t old_ping_sent; - clusterLink *link; - - fd = anetTcpNonBlockBindConnect(server.neterr, node->ip, - node->cport, NET_FIRST_BIND_ADDR); - if (fd == -1) { + clusterLink *link = createClusterLink(node); + link->conn = server.tls_cluster ? connCreateTLS() : connCreateSocket(); + connSetPrivateData(link->conn, link); + if (connConnect(link->conn, node->ip, node->cport, NET_FIRST_BIND_ADDR, + clusterLinkConnectHandler) == -1) { /* We got a synchronous error from connect before * clusterSendPing() had a chance to be called. * If node->ping_sent is zero, failure detection can't work, @@ -3399,37 +3500,11 @@ void clusterCron(void) { serverLog(LL_DEBUG, "Unable to connect to " "Cluster Node [%s]:%d -> %s", node->ip, node->cport, server.neterr); + + freeClusterLink(link); continue; } - link = createClusterLink(node); - link->fd = fd; node->link = link; - aeCreateFileEvent(server.el,link->fd,AE_READABLE, - clusterReadHandler,link); - /* Queue a PING in the new connection ASAP: this is crucial - * to avoid false positives in failure detection. - * - * If the node is flagged as MEET, we send a MEET message instead - * of a PING one, to force the receiver to add us in its node - * table. */ - old_ping_sent = node->ping_sent; - clusterSendPing(link, node->flags & CLUSTER_NODE_MEET ? - CLUSTERMSG_TYPE_MEET : CLUSTERMSG_TYPE_PING); - if (old_ping_sent) { - /* If there was an active ping before the link was - * disconnected, we want to restore the ping time, otherwise - * replaced by the clusterSendPing() call. */ - node->ping_sent = old_ping_sent; - } - /* We can clear the flag after the first packet is sent. - * If we'll never receive a PONG, we'll never send new packets - * to this node. Instead after the PONG is received and we - * are no longer in meet/handshake status, we want to send - * normal PING packets. */ - node->flags &= ~CLUSTER_NODE_MEET; - - serverLog(LL_DEBUG,"Connecting with Node %.40s at %s:%d", - node->name, node->ip, node->cport); } } dictReleaseIterator(di); @@ -3473,7 +3548,6 @@ void clusterCron(void) { while((de = dictNext(di)) != NULL) { clusterNode *node = dictGetVal(de); now = mstime(); /* Use an updated time at every iteration. */ - mstime_t delay; if (node->flags & (CLUSTER_NODE_MYSELF|CLUSTER_NODE_NOADDR|CLUSTER_NODE_HANDSHAKE)) @@ -3497,16 +3571,20 @@ void clusterCron(void) { this_slaves = okslaves; } - /* If we are waiting for the PONG more than half the cluster + /* If we are not receiving any data for more than half the cluster * timeout, reconnect the link: maybe there is a connection * issue even if the node is alive. */ + mstime_t ping_delay = now - node->ping_sent; + mstime_t data_delay = now - node->data_received; if (node->link && /* is connected */ now - node->link->ctime > server.cluster_node_timeout && /* was not already reconnected */ node->ping_sent && /* we already sent a ping */ node->pong_received < node->ping_sent && /* still waiting pong */ /* and we are waiting for the pong more than timeout/2 */ - now - node->ping_sent > server.cluster_node_timeout/2) + ping_delay > server.cluster_node_timeout/2 && + /* and in such interval we are not seeing any traffic at all. */ + data_delay > server.cluster_node_timeout/2) { /* Disconnect the link, it will be reconnected automatically. */ freeClusterLink(node->link); @@ -3538,12 +3616,18 @@ void clusterCron(void) { /* Check only if we have an active ping for this instance. */ if (node->ping_sent == 0) continue; - /* Compute the delay of the PONG. Note that if we already received - * the PONG, then node->ping_sent is zero, so can't reach this - * code at all. */ - delay = now - node->ping_sent; + /* Check if this node looks unreachable. + * Note that if we already received the PONG, then node->ping_sent + * is zero, so can't reach this code at all, so we don't risk of + * checking for a PONG delay if we didn't sent the PING. + * + * We also consider every incoming data as proof of liveness, since + * our cluster bus link is also used for data: under heavy data + * load pong delays are possible. */ + mstime_t node_delay = (ping_delay < data_delay) ? ping_delay : + data_delay; - if (delay > server.cluster_node_timeout) { + if (node_delay > server.cluster_node_timeout) { /* Timeout reached. Set the node as possibly failing if it is * not already in this state. */ if (!(node->flags & (CLUSTER_NODE_PFAIL|CLUSTER_NODE_FAIL))) { @@ -4128,18 +4212,24 @@ void clusterReplyMultiBulkSlots(client *c) { */ int num_masters = 0; - void *slot_replylen = addDeferredMultiBulkLength(c); + void *slot_replylen = addReplyDeferredLen(c); dictEntry *de; dictIterator *di = dictGetSafeIterator(server.cluster->nodes); while((de = dictNext(di)) != NULL) { clusterNode *node = dictGetVal(de); int j = 0, start = -1; + int i, nested_elements = 0; /* Skip slaves (that are iterated when producing the output of their * master) and masters not serving any slot. */ if (!nodeIsMaster(node) || node->numslots == 0) continue; + for(i = 0; i < node->numslaves; i++) { + if (nodeFailed(node->slaves[i])) continue; + nested_elements++; + } + for (j = 0; j < CLUSTER_SLOTS; j++) { int bit, i; @@ -4147,8 +4237,7 @@ void clusterReplyMultiBulkSlots(client *c) { if (start == -1) start = j; } if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) { - int nested_elements = 3; /* slots (2) + master addr (1). */ - void *nested_replylen = addDeferredMultiBulkLength(c); + addReplyArrayLen(c, nested_elements + 3); /* slots (2) + master addr (1). */ if (bit && j == CLUSTER_SLOTS-1) j++; @@ -4164,7 +4253,7 @@ void clusterReplyMultiBulkSlots(client *c) { start = -1; /* First node reply position is always the master */ - addReplyMultiBulkLen(c, 3); + addReplyArrayLen(c, 3); addReplyBulkCString(c, node->ip); addReplyLongLong(c, node->port); addReplyBulkCBuffer(c, node->name, CLUSTER_NAMELEN); @@ -4174,19 +4263,17 @@ void clusterReplyMultiBulkSlots(client *c) { /* This loop is copy/pasted from clusterGenNodeDescription() * with modifications for per-slot node aggregation */ if (nodeFailed(node->slaves[i])) continue; - addReplyMultiBulkLen(c, 3); + addReplyArrayLen(c, 3); addReplyBulkCString(c, node->slaves[i]->ip); addReplyLongLong(c, node->slaves[i]->port); addReplyBulkCBuffer(c, node->slaves[i]->name, CLUSTER_NAMELEN); - nested_elements++; } - setDeferredMultiBulkLength(c, nested_replylen, nested_elements); num_masters++; } } } dictReleaseIterator(di); - setDeferredMultiBulkLength(c, slot_replylen, num_masters); + setDeferredArrayLen(c, slot_replylen, num_masters); } void clusterCommand(client *c) { @@ -4206,7 +4293,7 @@ void clusterCommand(client *c) { "FORGET -- Remove a node from the cluster.", "GETKEYSINSLOT -- Return key names stored by current node in a slot.", "FLUSHSLOTS -- Delete current node own slots information.", -"INFO - Return onformation about the cluster.", +"INFO - Return information about the cluster.", "KEYSLOT -- Return the hash slot for .", "MEET [bus-port] -- Connect nodes into a working cluster.", "MYID -- Return the node id.", @@ -4217,6 +4304,7 @@ void clusterCommand(client *c) { "SET-config-epoch - Set config epoch of current node.", "SETSLOT (importing|migrating|stable|node ) -- Set slot state.", "REPLICAS -- Return replicas.", +"SAVECONFIG - Force saving cluster configuration on disk.", "SLOTS -- Return information about slots range mappings. Each range is made of:", " start, end, master and replicas IP addresses, ports and ids", NULL @@ -4252,12 +4340,9 @@ NULL } } else if (!strcasecmp(c->argv[1]->ptr,"nodes") && c->argc == 2) { /* CLUSTER NODES */ - robj *o; - sds ci = clusterGenNodesDescription(0); - - o = createObject(OBJ_STRING,ci); - addReplyBulk(c,o); - decrRefCount(o); + sds nodes = clusterGenNodesDescription(0); + addReplyVerbatim(c,nodes,sdslen(nodes),"txt"); + sdsfree(nodes); } else if (!strcasecmp(c->argv[1]->ptr,"myid") && c->argc == 2) { /* CLUSTER MYID */ addReplyBulkCBuffer(c,myself->name, CLUSTER_NAMELEN); @@ -4499,10 +4584,8 @@ NULL "cluster_stats_messages_received:%lld\r\n", tot_msg_received); /* Produce the reply protocol. */ - addReplySds(c,sdscatprintf(sdsempty(),"$%lu\r\n", - (unsigned long)sdslen(info))); - addReplySds(c,info); - addReply(c,shared.crlf); + addReplyVerbatim(c,info,sdslen(info),"txt"); + sdsfree(info); } else if (!strcasecmp(c->argv[1]->ptr,"saveconfig") && c->argc == 2) { int retval = clusterSaveConfig(1); @@ -4550,7 +4633,7 @@ NULL keys = zmalloc(sizeof(robj*)*maxkeys); numkeys = getKeysInSlot(slot, keys, maxkeys); - addReplyMultiBulkLen(c,numkeys); + addReplyArrayLen(c,numkeys); for (j = 0; j < numkeys; j++) { addReplyBulk(c,keys[j]); decrRefCount(keys[j]); @@ -4629,7 +4712,7 @@ NULL return; } - addReplyMultiBulkLen(c,n->numslaves); + addReplyArrayLen(c,n->numslaves); for (j = 0; j < n->numslaves; j++) { sds ni = clusterGenNodeDescription(n->slaves[j]); addReplyBulkCString(c,ni); @@ -4833,12 +4916,12 @@ int verifyDumpPayload(unsigned char *p, size_t len) { * DUMP is actually not used by Redis Cluster but it is the obvious * complement of RESTORE and can be useful for different applications. */ void dumpCommand(client *c) { - robj *o, *dumpobj; + robj *o; rio payload; /* Check if the key is here. */ if ((o = lookupKeyRead(c->db,c->argv[1])) == NULL) { - addReply(c,shared.nullbulk); + addReplyNull(c); return; } @@ -4846,9 +4929,7 @@ void dumpCommand(client *c) { createDumpPayload(&payload,o,c->argv[1]); /* Transfer to the client */ - dumpobj = createObject(OBJ_STRING,payload.io.buffer.ptr); - addReplyBulk(c,dumpobj); - decrRefCount(dumpobj); + addReplyBulkSds(c,payload.io.buffer.ptr); return; } @@ -4916,7 +4997,7 @@ void restoreCommand(client *c) { rioInitWithBuffer(&payload,c->argv[3]->ptr); if (((type = rdbLoadObjectType(&payload)) == -1) || - ((obj = rdbLoadObject(type,&payload,c->argv[1])) == NULL)) + ((obj = rdbLoadObject(type,&payload,c->argv[1]->ptr)) == NULL)) { addReplyError(c,"Bad data format"); return; @@ -4931,8 +5012,9 @@ void restoreCommand(client *c) { if (!absttl) ttl+=mstime(); setExpire(c,c->db,c->argv[1],ttl); } - objectSetLRUOrLFU(obj,lfu_freq,lru_idle,lru_clock); - signalModifiedKey(c->db,c->argv[1]); + objectSetLRUOrLFU(obj,lfu_freq,lru_idle,lru_clock,1000); + signalModifiedKey(c,c->db,c->argv[1]); + notifyKeyspaceEvent(NOTIFY_GENERIC,"restore",c->argv[1],c->db->id); addReply(c,shared.ok); server.dirty++; } @@ -4947,7 +5029,7 @@ void restoreCommand(client *c) { #define MIGRATE_SOCKET_CACHE_TTL 10 /* close cached sockets after 10 sec. */ typedef struct migrateCachedSocket { - int fd; + connection *conn; long last_dbid; time_t last_use_time; } migrateCachedSocket; @@ -4964,7 +5046,7 @@ typedef struct migrateCachedSocket { * should be called so that the connection will be created from scratch * the next time. */ migrateCachedSocket* migrateGetSocket(client *c, robj *host, robj *port, long timeout) { - int fd; + connection *conn; sds name = sdsempty(); migrateCachedSocket *cs; @@ -4984,34 +5066,27 @@ migrateCachedSocket* migrateGetSocket(client *c, robj *host, robj *port, long ti /* Too many items, drop one at random. */ dictEntry *de = dictGetRandomKey(server.migrate_cached_sockets); cs = dictGetVal(de); - close(cs->fd); + connClose(cs->conn); zfree(cs); dictDelete(server.migrate_cached_sockets,dictGetKey(de)); } /* Create the socket */ - fd = anetTcpNonBlockConnect(server.neterr,c->argv[1]->ptr, - atoi(c->argv[2]->ptr)); - if (fd == -1) { - sdsfree(name); - addReplyErrorFormat(c,"Can't connect to target node: %s", - server.neterr); - return NULL; - } - anetEnableTcpNoDelay(server.neterr,fd); - - /* Check if it connects within the specified timeout. */ - if ((aeWait(fd,AE_WRITABLE,timeout) & AE_WRITABLE) == 0) { - sdsfree(name); + conn = server.tls_cluster ? connCreateTLS() : connCreateSocket(); + if (connBlockingConnect(conn, c->argv[1]->ptr, atoi(c->argv[2]->ptr), timeout) + != C_OK) { addReplySds(c, sdsnew("-IOERR error or timeout connecting to the client\r\n")); - close(fd); + connClose(conn); + sdsfree(name); return NULL; } + connEnableTcpNoDelay(conn); /* Add to the cache and return it to the caller. */ cs = zmalloc(sizeof(*cs)); - cs->fd = fd; + cs->conn = conn; + cs->last_dbid = -1; cs->last_use_time = server.unixtime; dictAdd(server.migrate_cached_sockets,name,cs); @@ -5032,7 +5107,7 @@ void migrateCloseSocket(robj *host, robj *port) { return; } - close(cs->fd); + connClose(cs->conn); zfree(cs); dictDelete(server.migrate_cached_sockets,name); sdsfree(name); @@ -5046,7 +5121,7 @@ void migrateCloseTimedoutSockets(void) { migrateCachedSocket *cs = dictGetVal(de); if ((server.unixtime - cs->last_use_time) > MIGRATE_SOCKET_CACHE_TTL) { - close(cs->fd); + connClose(cs->conn); zfree(cs); dictDelete(server.migrate_cached_sockets,dictGetKey(de)); } @@ -5054,15 +5129,17 @@ void migrateCloseTimedoutSockets(void) { dictReleaseIterator(di); } -/* MIGRATE host port key dbid timeout [COPY | REPLACE | AUTH password] +/* MIGRATE host port key dbid timeout [COPY | REPLACE | AUTH password | + * AUTH2 username password] * * On in the multiple keys form: * - * MIGRATE host port "" dbid timeout [COPY | REPLACE | AUTH password] KEYS key1 - * key2 ... keyN */ + * MIGRATE host port "" dbid timeout [COPY | REPLACE | AUTH password | + * AUTH2 username password] KEYS key1 key2 ... keyN */ void migrateCommand(client *c) { migrateCachedSocket *cs; int copy = 0, replace = 0, j; + char *username = NULL; char *password = NULL; long timeout; long dbid; @@ -5080,7 +5157,7 @@ void migrateCommand(client *c) { /* Parse additional options */ for (j = 6; j < c->argc; j++) { - int moreargs = j < c->argc-1; + int moreargs = (c->argc-1) - j; if (!strcasecmp(c->argv[j]->ptr,"copy")) { copy = 1; } else if (!strcasecmp(c->argv[j]->ptr,"replace")) { @@ -5092,6 +5169,13 @@ void migrateCommand(client *c) { } j++; password = c->argv[j]->ptr; + } else if (!strcasecmp(c->argv[j]->ptr,"auth2")) { + if (moreargs < 2) { + addReply(c,shared.syntaxerr); + return; + } + username = c->argv[++j]->ptr; + password = c->argv[++j]->ptr; } else if (!strcasecmp(c->argv[j]->ptr,"keys")) { if (sdslen(c->argv[3]->ptr) != 0) { addReplyError(c, @@ -5152,8 +5236,13 @@ void migrateCommand(client *c) { /* Authentication */ if (password) { - serverAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',2)); + int arity = username ? 3 : 2; + serverAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',arity)); serverAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"AUTH",4)); + if (username) { + serverAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,username, + sdslen(username))); + } serverAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,password, sdslen(password))); } @@ -5228,7 +5317,7 @@ void migrateCommand(client *c) { while ((towrite = sdslen(buf)-pos) > 0) { towrite = (towrite > (64*1024) ? (64*1024) : towrite); - nwritten = syncWrite(cs->fd,buf+pos,towrite,timeout); + nwritten = connSyncWrite(cs->conn,buf+pos,towrite,timeout); if (nwritten != (signed)towrite) { write_error = 1; goto socket_err; @@ -5242,11 +5331,11 @@ void migrateCommand(client *c) { char buf2[1024]; /* Restore reply. */ /* Read the AUTH reply if needed. */ - if (password && syncReadLine(cs->fd, buf0, sizeof(buf0), timeout) <= 0) + if (password && connSyncReadLine(cs->conn, buf0, sizeof(buf0), timeout) <= 0) goto socket_err; /* Read the SELECT reply if needed. */ - if (select && syncReadLine(cs->fd, buf1, sizeof(buf1), timeout) <= 0) + if (select && connSyncReadLine(cs->conn, buf1, sizeof(buf1), timeout) <= 0) goto socket_err; /* Read the RESTORE replies. */ @@ -5261,7 +5350,7 @@ void migrateCommand(client *c) { if (!copy) newargv = zmalloc(sizeof(robj*)*(num_keys+1)); for (j = 0; j < num_keys; j++) { - if (syncReadLine(cs->fd, buf2, sizeof(buf2), timeout) <= 0) { + if (connSyncReadLine(cs->conn, buf2, sizeof(buf2), timeout) <= 0) { socket_error = 1; break; } @@ -5285,7 +5374,8 @@ void migrateCommand(client *c) { if (!copy) { /* No COPY option: remove the local key, signal the change. */ dbDelete(c->db,kv[j]); - signalModifiedKey(c->db,kv[j]); + signalModifiedKey(c,c->db,kv[j]); + notifyKeyspaceEvent(NOTIFY_GENERIC,"del",kv[j],c->db->id); server.dirty++; /* Populate the argument vector to replace the old one. */ @@ -5448,8 +5538,8 @@ void readwriteCommand(client *c) { * already "down" but it is fragile to rely on the update of the global state, * so we also handle it here. * - * CLUSTER_REDIR_DOWN_STATE if the cluster is down but the user attempts to - * execute a command that addresses one or more keys. */ + * CLUSTER_REDIR_DOWN_STATE and CLUSTER_REDIR_DOWN_RO_STATE if the cluster is + * down but the user attempts to execute a command that addresses one or more keys. */ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, int argc, int *hashslot, int *error_code) { clusterNode *n = NULL; robj *firstkey = NULL; @@ -5567,10 +5657,27 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in * without redirections or errors in all the cases. */ if (n == NULL) return myself; - /* Cluster is globally down but we got keys? We can't serve the request. */ + /* Cluster is globally down but we got keys? We only serve the request + * if it is a read command and when allow_reads_when_down is enabled. */ if (server.cluster->state != CLUSTER_OK) { - if (error_code) *error_code = CLUSTER_REDIR_DOWN_STATE; - return NULL; + if (!server.cluster_allow_reads_when_down) { + /* The cluster is configured to block commands when the + * cluster is down. */ + if (error_code) *error_code = CLUSTER_REDIR_DOWN_STATE; + return NULL; + } else if (!(cmd->flags & CMD_READONLY) && !(cmd->proc == evalCommand) + && !(cmd->proc == evalShaCommand)) + { + /* The cluster is configured to allow read only commands + * but this command is neither readonly, nor EVAL or + * EVALSHA. */ + if (error_code) *error_code = CLUSTER_REDIR_DOWN_RO_STATE; + return NULL; + } else { + /* Fall through and allow the command to be executed: + * this happens when server.cluster_allow_reads_when_down is + * true and the command is a readonly command or EVAL / EVALSHA. */ + } } /* Return the hashslot by reference. */ @@ -5639,6 +5746,8 @@ void clusterRedirectClient(client *c, clusterNode *n, int hashslot, int error_co addReplySds(c,sdsnew("-TRYAGAIN Multiple keys request during rehashing of slot\r\n")); } else if (error_code == CLUSTER_REDIR_DOWN_STATE) { addReplySds(c,sdsnew("-CLUSTERDOWN The cluster is down\r\n")); + } else if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) { + addReplySds(c,sdsnew("-CLUSTERDOWN The cluster is down and only accepts read commands\r\n")); } else if (error_code == CLUSTER_REDIR_DOWN_UNBOUND) { addReplySds(c,sdsnew("-CLUSTERDOWN Hash slot not served\r\n")); } else if (error_code == CLUSTER_REDIR_MOVED || @@ -5673,7 +5782,10 @@ int clusterRedirectBlockedClientIfNeeded(client *c) { dictEntry *de; dictIterator *di; - /* If the cluster is down, unblock the client with the right error. */ + /* If the cluster is down, unblock the client with the right error. + * If the cluster is configured to allow reads on cluster down, we + * still want to emit this error since a write will be required + * to unblock them which may never come. */ if (server.cluster->state == CLUSTER_FAIL) { clusterRedirectClient(c,NULL,0,CLUSTER_REDIR_DOWN_STATE); return 1; diff --git a/redis.submodule/src/cluster.h b/redis.submodule/src/cluster.h index 571b9c5..d3af4a3 100644 --- a/redis.submodule/src/cluster.h +++ b/redis.submodule/src/cluster.h @@ -13,15 +13,10 @@ /* The following defines are amount of time, sometimes expressed as * multiplicators of the node timeout value (when ending with MULT). */ -#define CLUSTER_DEFAULT_NODE_TIMEOUT 15000 -#define CLUSTER_DEFAULT_SLAVE_VALIDITY 10 /* Slave max data age factor. */ -#define CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE 1 -#define CLUSTER_DEFAULT_SLAVE_NO_FAILOVER 0 /* Failover by default. */ #define CLUSTER_FAIL_REPORT_VALIDITY_MULT 2 /* Fail report validity. */ #define CLUSTER_FAIL_UNDO_TIME_MULT 2 /* Undo fail if master is back. */ #define CLUSTER_FAIL_UNDO_TIME_ADD 10 /* Some additional time. */ #define CLUSTER_FAILOVER_DELAY 5 /* Seconds */ -#define CLUSTER_DEFAULT_MIGRATION_BARRIER 1 #define CLUSTER_MF_TIMEOUT 5000 /* Milliseconds to do a manual failover. */ #define CLUSTER_MF_PAUSE_MULT 2 /* Master pause manual failover mult. */ #define CLUSTER_SLAVE_MIGRATION_DELAY 5000 /* Delay for slave migration. */ @@ -34,13 +29,14 @@ #define CLUSTER_REDIR_MOVED 4 /* -MOVED redirection required. */ #define CLUSTER_REDIR_DOWN_STATE 5 /* -CLUSTERDOWN, global state. */ #define CLUSTER_REDIR_DOWN_UNBOUND 6 /* -CLUSTERDOWN, unbound slot. */ +#define CLUSTER_REDIR_DOWN_RO_STATE 7 /* -CLUSTERDOWN, allow reads. */ struct clusterNode; /* clusterLink encapsulates everything needed to talk with a remote node. */ typedef struct clusterLink { mstime_t ctime; /* Link creation time */ - int fd; /* TCP socket file descriptor */ + connection *conn; /* Connection to remote node */ sds sndbuf; /* Packet send buffer */ sds rcvbuf; /* Packet reception buffer */ struct clusterNode *node; /* Node related to this link if any, or NULL */ @@ -128,6 +124,7 @@ typedef struct clusterNode { tables. */ mstime_t ping_sent; /* Unix time we sent latest ping */ mstime_t pong_received; /* Unix time we received the pong */ + mstime_t data_received; /* Unix time we received any data */ mstime_t fail_time; /* Unix time when FAIL flag was set */ mstime_t voted_time; /* Last time we voted for a slave of this master */ mstime_t repl_offset_time; /* Unix time we received offset for this node */ diff --git a/redis.submodule/src/config.c b/redis.submodule/src/config.c index ba8e17e..6485459 100644 --- a/redis.submodule/src/config.c +++ b/redis.submodule/src/config.c @@ -91,6 +91,13 @@ configEnum aof_fsync_enum[] = { {NULL, 0} }; +configEnum repl_diskless_load_enum[] = { + {"disabled", REPL_DISKLESS_LOAD_DISABLED}, + {"on-empty-db", REPL_DISKLESS_LOAD_WHEN_DB_EMPTY}, + {"swapdb", REPL_DISKLESS_LOAD_SWAPDB}, + {NULL, 0} +}; + /* Output buffer limits presets. */ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { {0, 0, 0}, /* normal */ @@ -98,6 +105,110 @@ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { {1024*1024*32, 1024*1024*8, 60} /* pubsub */ }; +/* Generic config infrastructure function pointers + * int is_valid_fn(val, err) + * Return 1 when val is valid, and 0 when invalid. + * Optionally set err to a static error string. + * int update_fn(val, prev, err) + * This function is called only for CONFIG SET command (not at config file parsing) + * It is called after the actual config is applied, + * Return 1 for success, and 0 for failure. + * Optionally set err to a static error string. + * On failure the config change will be reverted. + */ + +/* Configuration values that require no special handling to set, get, load or + * rewrite. */ +typedef struct boolConfigData { + int *config; /* The pointer to the server config this value is stored in */ + const int default_value; /* The default value of the config on rewrite */ + int (*is_valid_fn)(int val, char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(int val, int prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */ +} boolConfigData; + +typedef struct stringConfigData { + char **config; /* Pointer to the server config this value is stored in. */ + const char *default_value; /* Default value of the config on rewrite. */ + int (*is_valid_fn)(char* val, char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(char* val, char* prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */ + int convert_empty_to_null; /* Boolean indicating if empty strings should + be stored as a NULL value. */ +} stringConfigData; + +typedef struct enumConfigData { + int *config; /* The pointer to the server config this value is stored in */ + configEnum *enum_value; /* The underlying enum type this data represents */ + const int default_value; /* The default value of the config on rewrite */ + int (*is_valid_fn)(int val, char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(int val, int prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */ +} enumConfigData; + +typedef enum numericType { + NUMERIC_TYPE_INT, + NUMERIC_TYPE_UINT, + NUMERIC_TYPE_LONG, + NUMERIC_TYPE_ULONG, + NUMERIC_TYPE_LONG_LONG, + NUMERIC_TYPE_ULONG_LONG, + NUMERIC_TYPE_SIZE_T, + NUMERIC_TYPE_SSIZE_T, + NUMERIC_TYPE_OFF_T, + NUMERIC_TYPE_TIME_T, +} numericType; + +typedef struct numericConfigData { + union { + int *i; + unsigned int *ui; + long *l; + unsigned long *ul; + long long *ll; + unsigned long long *ull; + size_t *st; + ssize_t *sst; + off_t *ot; + time_t *tt; + } config; /* The pointer to the numeric config this value is stored in */ + int is_memory; /* Indicates if this value can be loaded as a memory value */ + numericType numeric_type; /* An enum indicating the type of this value */ + long long lower_bound; /* The lower bound of this numeric value */ + long long upper_bound; /* The upper bound of this numeric value */ + const long long default_value; /* The default value of the config on rewrite */ + int (*is_valid_fn)(long long val, char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(long long val, long long prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */ +} numericConfigData; + +typedef union typeData { + boolConfigData yesno; + stringConfigData string; + enumConfigData enumd; + numericConfigData numeric; +} typeData; + +typedef struct typeInterface { + /* Called on server start, to init the server with default value */ + void (*init)(typeData data); + /* Called on server start, should return 1 on success, 0 on error and should set err */ + int (*load)(typeData data, sds *argc, int argv, char **err); + /* Called on server startup and CONFIG SET, returns 1 on success, 0 on error + * and can set a verbose err string, update is true when called from CONFIG SET */ + int (*set)(typeData data, sds value, int update, char **err); + /* Called on CONFIG GET, required to add output to the client */ + void (*get)(client *c, typeData data); + /* Called on CONFIG REWRITE, required to rewrite the config state */ + void (*rewrite)(typeData data, const char *name, struct rewriteConfigState *state); +} typeInterface; + +typedef struct standardConfig { + const char *name; /* The user visible name of this config */ + const char *alias; /* An alias that can also be used for this config */ + const int modifiable; /* Can this value be updated by CONFIG SET? */ + typeInterface interface; /* The function pointers that define the type interface */ + typeData data; /* The type specific data exposed used by the interface */ +} standardConfig; + +standardConfig configs[]; + /*----------------------------------------------------------------------------- * Enum access functions *----------------------------------------------------------------------------*/ @@ -169,6 +280,12 @@ void queueLoadModule(sds path, sds *argv, int argc) { listAddNodeTail(server.loadmodule_queue,loadmod); } +void initConfigValues() { + for (standardConfig *config = configs; config->name != NULL; config++) { + config->interface.init(config->data); + } +} + void loadServerConfigFromString(char *config) { char *err = NULL; int linenum = 0, totlines, i; @@ -201,42 +318,44 @@ void loadServerConfigFromString(char *config) { } sdstolower(argv[0]); - /* Execute config directives */ - if (!strcasecmp(argv[0],"timeout") && argc == 2) { - server.maxidletime = atoi(argv[1]); - if (server.maxidletime < 0) { - err = "Invalid timeout value"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"tcp-keepalive") && argc == 2) { - server.tcpkeepalive = atoi(argv[1]); - if (server.tcpkeepalive < 0) { - err = "Invalid tcp-keepalive value"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"protected-mode") && argc == 2) { - if ((server.protected_mode = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"port") && argc == 2) { - server.port = atoi(argv[1]); - if (server.port < 0 || server.port > 65535) { - err = "Invalid port"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"tcp-backlog") && argc == 2) { - server.tcp_backlog = atoi(argv[1]); - if (server.tcp_backlog < 0) { - err = "Invalid backlog value"; goto loaderr; + /* Iterate the configs that are standard */ + int match = 0; + for (standardConfig *config = configs; config->name != NULL; config++) { + if ((!strcasecmp(argv[0],config->name) || + (config->alias && !strcasecmp(argv[0],config->alias)))) + { + if (argc != 2) { + err = "wrong number of arguments"; + goto loaderr; + } + if (!config->interface.set(config->data, argv[1], 0, &err)) { + goto loaderr; + } + + match = 1; + break; } - } else if (!strcasecmp(argv[0],"bind") && argc >= 2) { + } + + if (match) { + sdsfreesplitres(argv,argc); + continue; + } + + /* Execute config directives */ + if (!strcasecmp(argv[0],"bind") && argc >= 2) { int j, addresses = argc-1; if (addresses > CONFIG_BINDADDR_MAX) { err = "Too many bind addresses specified"; goto loaderr; } + /* Free old bind addresses */ + for (j = 0; j < server.bindaddr_count; j++) { + zfree(server.bindaddr[j]); + } for (j = 0; j < addresses; j++) server.bindaddr[j] = zstrdup(argv[j+1]); server.bindaddr_count = addresses; - } else if (!strcasecmp(argv[0],"unixsocket") && argc == 2) { - server.unixsocket = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"unixsocketperm") && argc == 2) { errno = 0; server.unixsocketperm = (mode_t)strtol(argv[1], NULL, 8); @@ -260,13 +379,6 @@ void loadServerConfigFromString(char *config) { argv[1], strerror(errno)); exit(1); } - } else if (!strcasecmp(argv[0],"loglevel") && argc == 2) { - server.verbosity = configEnumGetValue(loglevel_enum,argv[1]); - if (server.verbosity == INT_MIN) { - err = "Invalid log level. " - "Must be one of debug, verbose, notice, warning"; - goto loaderr; - } } else if (!strcasecmp(argv[0],"logfile") && argc == 2) { FILE *logfp; @@ -283,327 +395,35 @@ void loadServerConfigFromString(char *config) { } fclose(logfp); } - } else if (!strcasecmp(argv[0],"always-show-logo") && argc == 2) { - if ((server.always_show_logo = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"syslog-enabled") && argc == 2) { - if ((server.syslog_enabled = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"syslog-ident") && argc == 2) { - if (server.syslog_ident) zfree(server.syslog_ident); - server.syslog_ident = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"syslog-facility") && argc == 2) { - server.syslog_facility = - configEnumGetValue(syslog_facility_enum,argv[1]); - if (server.syslog_facility == INT_MIN) { - err = "Invalid log facility. Must be one of USER or between LOCAL0-LOCAL7"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"databases") && argc == 2) { - server.dbnum = atoi(argv[1]); - if (server.dbnum < 1) { - err = "Invalid number of databases"; goto loaderr; - } } else if (!strcasecmp(argv[0],"include") && argc == 2) { loadServerConfig(argv[1],NULL); - } else if (!strcasecmp(argv[0],"maxclients") && argc == 2) { - server.maxclients = atoi(argv[1]); - if (server.maxclients < 1) { - err = "Invalid max clients limit"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"maxmemory") && argc == 2) { - server.maxmemory = memtoll(argv[1],NULL); - } else if (!strcasecmp(argv[0],"maxmemory-policy") && argc == 2) { - server.maxmemory_policy = - configEnumGetValue(maxmemory_policy_enum,argv[1]); - if (server.maxmemory_policy == INT_MIN) { - err = "Invalid maxmemory policy"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"maxmemory-samples") && argc == 2) { - server.maxmemory_samples = atoi(argv[1]); - if (server.maxmemory_samples <= 0) { - err = "maxmemory-samples must be 1 or greater"; - goto loaderr; - } - } else if ((!strcasecmp(argv[0],"proto-max-bulk-len")) && argc == 2) { - server.proto_max_bulk_len = memtoll(argv[1],NULL); } else if ((!strcasecmp(argv[0],"client-query-buffer-limit")) && argc == 2) { - server.client_max_querybuf_len = memtoll(argv[1],NULL); - } else if (!strcasecmp(argv[0],"lfu-log-factor") && argc == 2) { - server.lfu_log_factor = atoi(argv[1]); - if (server.lfu_log_factor < 0) { - err = "lfu-log-factor must be 0 or greater"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"lfu-decay-time") && argc == 2) { - server.lfu_decay_time = atoi(argv[1]); - if (server.lfu_decay_time < 0) { - err = "lfu-decay-time must be 0 or greater"; - goto loaderr; - } + server.client_max_querybuf_len = memtoll(argv[1],NULL); } else if ((!strcasecmp(argv[0],"slaveof") || !strcasecmp(argv[0],"replicaof")) && argc == 3) { slaveof_linenum = linenum; server.masterhost = sdsnew(argv[1]); server.masterport = atoi(argv[2]); server.repl_state = REPL_STATE_CONNECT; - } else if ((!strcasecmp(argv[0],"repl-ping-slave-period") || - !strcasecmp(argv[0],"repl-ping-replica-period")) && - argc == 2) - { - server.repl_ping_slave_period = atoi(argv[1]); - if (server.repl_ping_slave_period <= 0) { - err = "repl-ping-replica-period must be 1 or greater"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"repl-timeout") && argc == 2) { - server.repl_timeout = atoi(argv[1]); - if (server.repl_timeout <= 0) { - err = "repl-timeout must be 1 or greater"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"repl-disable-tcp-nodelay") && argc==2) { - if ((server.repl_disable_tcp_nodelay = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"repl-diskless-sync") && argc==2) { - if ((server.repl_diskless_sync = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"repl-diskless-sync-delay") && argc==2) { - server.repl_diskless_sync_delay = atoi(argv[1]); - if (server.repl_diskless_sync_delay < 0) { - err = "repl-diskless-sync-delay can't be negative"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"repl-backlog-size") && argc == 2) { - long long size = memtoll(argv[1],NULL); - if (size <= 0) { - err = "repl-backlog-size must be 1 or greater."; - goto loaderr; - } - resizeReplicationBacklog(size); - } else if (!strcasecmp(argv[0],"repl-backlog-ttl") && argc == 2) { - server.repl_backlog_time_limit = atoi(argv[1]); - if (server.repl_backlog_time_limit < 0) { - err = "repl-backlog-ttl can't be negative "; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"masterauth") && argc == 2) { - zfree(server.masterauth); - server.masterauth = argv[1][0] ? zstrdup(argv[1]) : NULL; - } else if ((!strcasecmp(argv[0],"slave-serve-stale-data") || - !strcasecmp(argv[0],"replica-serve-stale-data")) - && argc == 2) - { - if ((server.repl_serve_stale_data = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if ((!strcasecmp(argv[0],"slave-read-only") || - !strcasecmp(argv[0],"replica-read-only")) - && argc == 2) - { - if ((server.repl_slave_ro = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if ((!strcasecmp(argv[0],"slave-ignore-maxmemory") || - !strcasecmp(argv[0],"replica-ignore-maxmemory")) - && argc == 2) - { - if ((server.repl_slave_ignore_maxmemory = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"rdbcompression") && argc == 2) { - if ((server.rdb_compression = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"rdbchecksum") && argc == 2) { - if ((server.rdb_checksum = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"activerehashing") && argc == 2) { - if ((server.activerehashing = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"lazyfree-lazy-eviction") && argc == 2) { - if ((server.lazyfree_lazy_eviction = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"lazyfree-lazy-expire") && argc == 2) { - if ((server.lazyfree_lazy_expire = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"lazyfree-lazy-server-del") && argc == 2){ - if ((server.lazyfree_lazy_server_del = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if ((!strcasecmp(argv[0],"slave-lazy-flush") || - !strcasecmp(argv[0],"replica-lazy-flush")) && argc == 2) - { - if ((server.repl_slave_lazy_flush = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"activedefrag") && argc == 2) { - if ((server.active_defrag_enabled = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - if (server.active_defrag_enabled) { -#ifndef HAVE_DEFRAG - err = "active defrag can't be enabled without proper jemalloc support"; goto loaderr; -#endif - } - } else if (!strcasecmp(argv[0],"daemonize") && argc == 2) { - if ((server.daemonize = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"dynamic-hz") && argc == 2) { - if ((server.dynamic_hz = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"hz") && argc == 2) { - server.config_hz = atoi(argv[1]); - if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; - if (server.config_hz > CONFIG_MAX_HZ) server.config_hz = CONFIG_MAX_HZ; - } else if (!strcasecmp(argv[0],"appendonly") && argc == 2) { - int yes; - - if ((yes = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - server.aof_state = yes ? AOF_ON : AOF_OFF; - } else if (!strcasecmp(argv[0],"appendfilename") && argc == 2) { - if (!pathIsBaseName(argv[1])) { - err = "appendfilename can't be a path, just a filename"; - goto loaderr; - } - zfree(server.aof_filename); - server.aof_filename = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"no-appendfsync-on-rewrite") - && argc == 2) { - if ((server.aof_no_fsync_on_rewrite= yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"appendfsync") && argc == 2) { - server.aof_fsync = configEnumGetValue(aof_fsync_enum,argv[1]); - if (server.aof_fsync == INT_MIN) { - err = "argument must be 'no', 'always' or 'everysec'"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"auto-aof-rewrite-percentage") && - argc == 2) - { - server.aof_rewrite_perc = atoi(argv[1]); - if (server.aof_rewrite_perc < 0) { - err = "Invalid negative percentage for AOF auto rewrite"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"auto-aof-rewrite-min-size") && - argc == 2) - { - server.aof_rewrite_min_size = memtoll(argv[1],NULL); - } else if (!strcasecmp(argv[0],"aof-rewrite-incremental-fsync") && - argc == 2) - { - if ((server.aof_rewrite_incremental_fsync = - yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"rdb-save-incremental-fsync") && - argc == 2) - { - if ((server.rdb_save_incremental_fsync = - yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"aof-load-truncated") && argc == 2) { - if ((server.aof_load_truncated = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"aof-use-rdb-preamble") && argc == 2) { - if ((server.aof_use_rdb_preamble = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } } else if (!strcasecmp(argv[0],"requirepass") && argc == 2) { if (strlen(argv[1]) > CONFIG_AUTHPASS_MAX_LEN) { err = "Password is longer than CONFIG_AUTHPASS_MAX_LEN"; goto loaderr; } - server.requirepass = argv[1][0] ? zstrdup(argv[1]) : NULL; - } else if (!strcasecmp(argv[0],"pidfile") && argc == 2) { - zfree(server.pidfile); - server.pidfile = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"dbfilename") && argc == 2) { - if (!pathIsBaseName(argv[1])) { - err = "dbfilename can't be a path, just a filename"; - goto loaderr; - } - zfree(server.rdb_filename); - server.rdb_filename = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"active-defrag-threshold-lower") && argc == 2) { - server.active_defrag_threshold_lower = atoi(argv[1]); - if (server.active_defrag_threshold_lower < 0 || - server.active_defrag_threshold_lower > 1000) { - err = "active-defrag-threshold-lower must be between 0 and 1000"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-threshold-upper") && argc == 2) { - server.active_defrag_threshold_upper = atoi(argv[1]); - if (server.active_defrag_threshold_upper < 0 || - server.active_defrag_threshold_upper > 1000) { - err = "active-defrag-threshold-upper must be between 0 and 1000"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-ignore-bytes") && argc == 2) { - server.active_defrag_ignore_bytes = memtoll(argv[1], NULL); - if (server.active_defrag_ignore_bytes <= 0) { - err = "active-defrag-ignore-bytes must above 0"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-cycle-min") && argc == 2) { - server.active_defrag_cycle_min = atoi(argv[1]); - if (server.active_defrag_cycle_min < 1 || server.active_defrag_cycle_min > 99) { - err = "active-defrag-cycle-min must be between 1 and 99"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-cycle-max") && argc == 2) { - server.active_defrag_cycle_max = atoi(argv[1]); - if (server.active_defrag_cycle_max < 1 || server.active_defrag_cycle_max > 99) { - err = "active-defrag-cycle-max must be between 1 and 99"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-max-scan-fields") && argc == 2) { - server.active_defrag_max_scan_fields = strtoll(argv[1],NULL,10); - if (server.active_defrag_max_scan_fields < 1) { - err = "active-defrag-max-scan-fields must be positive"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"hash-max-ziplist-entries") && argc == 2) { - server.hash_max_ziplist_entries = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"hash-max-ziplist-value") && argc == 2) { - server.hash_max_ziplist_value = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"stream-node-max-bytes") && argc == 2) { - server.stream_node_max_bytes = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"stream-node-max-entries") && argc == 2) { - server.stream_node_max_entries = atoi(argv[1]); + /* The old "requirepass" directive just translates to setting + * a password to the default user. The only thing we do + * additionally is to remember the cleartext password in this + * case, for backward compatibility with Redis <= 5. */ + ACLSetUser(DefaultUser,"resetpass",-1); + sds aclop = sdscatprintf(sdsempty(),">%s",argv[1]); + ACLSetUser(DefaultUser,aclop,sdslen(aclop)); + sdsfree(aclop); + sdsfree(server.requirepass); + server.requirepass = sdsnew(argv[1]); } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){ /* DEAD OPTION */ } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2) { /* DEAD OPTION */ - } else if (!strcasecmp(argv[0],"list-max-ziplist-size") && argc == 2) { - server.list_max_ziplist_size = atoi(argv[1]); - } else if (!strcasecmp(argv[0],"list-compress-depth") && argc == 2) { - server.list_compress_depth = atoi(argv[1]); - } else if (!strcasecmp(argv[0],"set-max-intset-entries") && argc == 2) { - server.set_max_intset_entries = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"zset-max-ziplist-entries") && argc == 2) { - server.zset_max_ziplist_entries = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"zset-max-ziplist-value") && argc == 2) { - server.zset_max_ziplist_value = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"hll-sparse-max-bytes") && argc == 2) { - server.hll_sparse_max_bytes = memtoll(argv[1], NULL); } else if (!strcasecmp(argv[0],"rename-command") && argc == 3) { struct redisCommand *cmd = lookupCommand(argv[1]); int retval; @@ -628,88 +448,9 @@ void loadServerConfigFromString(char *config) { err = "Target command name already exists"; goto loaderr; } } - } else if (!strcasecmp(argv[0],"cluster-enabled") && argc == 2) { - if ((server.cluster_enabled = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } } else if (!strcasecmp(argv[0],"cluster-config-file") && argc == 2) { zfree(server.cluster_configfile); server.cluster_configfile = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"cluster-announce-ip") && argc == 2) { - zfree(server.cluster_announce_ip); - server.cluster_announce_ip = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"cluster-announce-port") && argc == 2) { - server.cluster_announce_port = atoi(argv[1]); - if (server.cluster_announce_port < 0 || - server.cluster_announce_port > 65535) - { - err = "Invalid port"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"cluster-announce-bus-port") && - argc == 2) - { - server.cluster_announce_bus_port = atoi(argv[1]); - if (server.cluster_announce_bus_port < 0 || - server.cluster_announce_bus_port > 65535) - { - err = "Invalid port"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"cluster-require-full-coverage") && - argc == 2) - { - if ((server.cluster_require_full_coverage = yesnotoi(argv[1])) == -1) - { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"cluster-node-timeout") && argc == 2) { - server.cluster_node_timeout = strtoll(argv[1],NULL,10); - if (server.cluster_node_timeout <= 0) { - err = "cluster node timeout must be 1 or greater"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"cluster-migration-barrier") - && argc == 2) - { - server.cluster_migration_barrier = atoi(argv[1]); - if (server.cluster_migration_barrier < 0) { - err = "cluster migration barrier must zero or positive"; - goto loaderr; - } - } else if ((!strcasecmp(argv[0],"cluster-slave-validity-factor") || - !strcasecmp(argv[0],"cluster-replica-validity-factor")) - && argc == 2) - { - server.cluster_slave_validity_factor = atoi(argv[1]); - if (server.cluster_slave_validity_factor < 0) { - err = "cluster replica validity factor must be zero or positive"; - goto loaderr; - } - } else if ((!strcasecmp(argv[0],"cluster-slave-no-failover") || - !strcasecmp(argv[0],"cluster-replica-no-failover")) && - argc == 2) - { - server.cluster_slave_no_failover = yesnotoi(argv[1]); - if (server.cluster_slave_no_failover == -1) { - err = "argument must be 'yes' or 'no'"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { - server.lua_time_limit = strtoll(argv[1],NULL,10); - } else if (!strcasecmp(argv[0],"lua-replicate-commands") && argc == 2) { - server.lua_always_replicate_commands = yesnotoi(argv[1]); - } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && - argc == 2) - { - server.slowlog_log_slower_than = strtoll(argv[1],NULL,10); - } else if (!strcasecmp(argv[0],"latency-monitor-threshold") && - argc == 2) - { - server.latency_monitor_threshold = strtoll(argv[1],NULL,10); - if (server.latency_monitor_threshold < 0) { - err = "The latency threshold can't be negative"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"slowlog-max-len") && argc == 2) { - server.slowlog_max_len = strtoll(argv[1],NULL,10); } else if (!strcasecmp(argv[0],"client-output-buffer-limit") && argc == 5) { @@ -732,43 +473,6 @@ void loadServerConfigFromString(char *config) { server.client_obuf_limits[class].hard_limit_bytes = hard; server.client_obuf_limits[class].soft_limit_bytes = soft; server.client_obuf_limits[class].soft_limit_seconds = soft_seconds; - } else if (!strcasecmp(argv[0],"stop-writes-on-bgsave-error") && - argc == 2) { - if ((server.stop_writes_on_bgsave_err = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if ((!strcasecmp(argv[0],"slave-priority") || - !strcasecmp(argv[0],"replica-priority")) && argc == 2) - { - server.slave_priority = atoi(argv[1]); - } else if ((!strcasecmp(argv[0],"slave-announce-ip") || - !strcasecmp(argv[0],"replica-announce-ip")) && argc == 2) - { - zfree(server.slave_announce_ip); - server.slave_announce_ip = zstrdup(argv[1]); - } else if ((!strcasecmp(argv[0],"slave-announce-port") || - !strcasecmp(argv[0],"replica-announce-port")) && argc == 2) - { - server.slave_announce_port = atoi(argv[1]); - if (server.slave_announce_port < 0 || - server.slave_announce_port > 65535) - { - err = "Invalid port"; goto loaderr; - } - } else if ((!strcasecmp(argv[0],"min-slaves-to-write") || - !strcasecmp(argv[0],"min-replicas-to-write")) && argc == 2) - { - server.repl_min_slaves_to_write = atoi(argv[1]); - if (server.repl_min_slaves_to_write < 0) { - err = "Invalid value for min-replicas-to-write."; goto loaderr; - } - } else if ((!strcasecmp(argv[0],"min-slaves-max-lag") || - !strcasecmp(argv[0],"min-replicas-max-lag")) && argc == 2) - { - server.repl_min_slaves_max_lag = atoi(argv[1]); - if (server.repl_min_slaves_max_lag < 0) { - err = "Invalid value for min-replicas-max-lag."; goto loaderr; - } } else if (!strcasecmp(argv[0],"notify-keyspace-events") && argc == 2) { int flags = keyspaceEventsStringToFlags(argv[1]); @@ -777,13 +481,14 @@ void loadServerConfigFromString(char *config) { goto loaderr; } server.notify_keyspace_events = flags; - } else if (!strcasecmp(argv[0],"supervised") && argc == 2) { - server.supervised_mode = - configEnumGetValue(supervised_mode_enum,argv[1]); - - if (server.supervised_mode == INT_MIN) { - err = "Invalid option for 'supervised'. " - "Allowed values: 'upstart', 'systemd', 'auto', or 'no'"; + } else if (!strcasecmp(argv[0],"user") && argc >= 2) { + int argc_err; + if (ACLAppendUserForLoading(argv,argc,&argc_err) == C_ERR) { + char buf[1024]; + char *errmsg = ACLSetUserStringError(); + snprintf(buf,sizeof(buf),"Error in user declaration '%s': %s", + argv[argc_err],errmsg); + err = buf; goto loaderr; } } else if (!strcasecmp(argv[0],"loadmodule") && argc >= 2) { @@ -817,7 +522,8 @@ void loadServerConfigFromString(char *config) { return; loaderr: - fprintf(stderr, "\n*** FATAL CONFIG FILE ERROR ***\n"); + fprintf(stderr, "\n*** FATAL CONFIG FILE ERROR (Redis %s) ***\n", + REDIS_VERSION); fprintf(stderr, "Reading the configuration file, at line %d\n", linenum); fprintf(stderr, ">>> '%s'\n", lines[i]); fprintf(stderr, "%s\n", err); @@ -884,12 +590,6 @@ void loadServerConfig(char *filename, char *options) { if (err || ll < 0) goto badfmt; \ _var = ll; -#define config_set_enum_field(_name,_var,_enumvar) \ - } else if (!strcasecmp(c->argv[2]->ptr,_name)) { \ - int enumval = configEnumGetValue(_enumvar,o->ptr); \ - if (enumval == INT_MIN) goto badfmt; \ - _var = enumval; - #define config_set_special_field(_name) \ } else if (!strcasecmp(c->argv[2]->ptr,_name)) { @@ -903,69 +603,39 @@ void configSetCommand(client *c) { robj *o; long long ll; int err; + char *errstr = NULL; serverAssertWithInfo(c,c->argv[2],sdsEncodedObject(c->argv[2])); serverAssertWithInfo(c,c->argv[3],sdsEncodedObject(c->argv[3])); o = c->argv[3]; + /* Iterate the configs that are standard */ + for (standardConfig *config = configs; config->name != NULL; config++) { + if(config->modifiable && (!strcasecmp(c->argv[2]->ptr,config->name) || + (config->alias && !strcasecmp(c->argv[2]->ptr,config->alias)))) + { + if (!config->interface.set(config->data,o->ptr,1,&errstr)) { + goto badfmt; + } + addReply(c,shared.ok); + return; + } + } + if (0) { /* this starts the config_set macros else-if chain. */ /* Special fields that can't be handled with general macros. */ - config_set_special_field("dbfilename") { - if (!pathIsBaseName(o->ptr)) { - addReplyError(c, "dbfilename can't be a path, just a filename"); - return; - } - zfree(server.rdb_filename); - server.rdb_filename = zstrdup(o->ptr); - } config_set_special_field("requirepass") { + config_set_special_field("requirepass") { if (sdslen(o->ptr) > CONFIG_AUTHPASS_MAX_LEN) goto badfmt; - zfree(server.requirepass); - server.requirepass = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; - } config_set_special_field("masterauth") { - zfree(server.masterauth); - server.masterauth = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; - } config_set_special_field("cluster-announce-ip") { - zfree(server.cluster_announce_ip); - server.cluster_announce_ip = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; - } config_set_special_field("maxclients") { - int orig_value = server.maxclients; - - if (getLongLongFromObject(o,&ll) == C_ERR || ll < 1) goto badfmt; - - /* Try to check if the OS is capable of supporting so many FDs. */ - server.maxclients = ll; - if (ll > orig_value) { - adjustOpenFilesLimit(); - if (server.maxclients != ll) { - addReplyErrorFormat(c,"The operating system is not able to handle the specified number of clients, try with %d", server.maxclients); - server.maxclients = orig_value; - return; - } - if ((unsigned int) aeGetSetSize(server.el) < - server.maxclients + CONFIG_FDSET_INCR) - { - if (aeResizeSetSize(server.el, - server.maxclients + CONFIG_FDSET_INCR) == AE_ERR) - { - addReplyError(c,"The event loop API used by Redis is not able to handle the specified number of clients"); - server.maxclients = orig_value; - return; - } - } - } - } config_set_special_field("appendonly") { - int enable = yesnotoi(o->ptr); - - if (enable == -1) goto badfmt; - if (enable == 0 && server.aof_state != AOF_OFF) { - stopAppendOnly(); - } else if (enable && server.aof_state == AOF_OFF) { - if (startAppendOnly() == C_ERR) { - addReplyError(c, - "Unable to turn on AOF. Check server logs."); - return; - } - } + /* The old "requirepass" directive just translates to setting + * a password to the default user. The only thing we do + * additionally is to remember the cleartext password in this + * case, for backward compatibility with Redis <= 5. */ + ACLSetUser(DefaultUser,"resetpass",-1); + sds aclop = sdscatprintf(sdsempty(),">%s",(char*)o->ptr); + ACLSetUser(DefaultUser,aclop,sdslen(aclop)); + sdsfree(aclop); + sdsfree(server.requirepass); + server.requirepass = sdsnew(o->ptr); } config_set_special_field("save") { int vlen, j; sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen); @@ -1056,219 +726,19 @@ void configSetCommand(client *c) { if (flags == -1) goto badfmt; server.notify_keyspace_events = flags; - } config_set_special_field_with_alias("slave-announce-ip", - "replica-announce-ip") - { - zfree(server.slave_announce_ip); - server.slave_announce_ip = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; - - /* Boolean fields. - * config_set_bool_field(name,var). */ - } config_set_bool_field( - "rdbcompression", server.rdb_compression) { - } config_set_bool_field( - "repl-disable-tcp-nodelay",server.repl_disable_tcp_nodelay) { - } config_set_bool_field( - "repl-diskless-sync",server.repl_diskless_sync) { - } config_set_bool_field( - "cluster-require-full-coverage",server.cluster_require_full_coverage) { - } config_set_bool_field( - "cluster-slave-no-failover",server.cluster_slave_no_failover) { - } config_set_bool_field( - "cluster-replica-no-failover",server.cluster_slave_no_failover) { - } config_set_bool_field( - "aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync) { - } config_set_bool_field( - "rdb-save-incremental-fsync",server.rdb_save_incremental_fsync) { - } config_set_bool_field( - "aof-load-truncated",server.aof_load_truncated) { - } config_set_bool_field( - "aof-use-rdb-preamble",server.aof_use_rdb_preamble) { - } config_set_bool_field( - "slave-serve-stale-data",server.repl_serve_stale_data) { - } config_set_bool_field( - "replica-serve-stale-data",server.repl_serve_stale_data) { - } config_set_bool_field( - "slave-read-only",server.repl_slave_ro) { - } config_set_bool_field( - "replica-read-only",server.repl_slave_ro) { - } config_set_bool_field( - "slave-ignore-maxmemory",server.repl_slave_ignore_maxmemory) { - } config_set_bool_field( - "replica-ignore-maxmemory",server.repl_slave_ignore_maxmemory) { - } config_set_bool_field( - "activerehashing",server.activerehashing) { - } config_set_bool_field( - "activedefrag",server.active_defrag_enabled) { -#ifndef HAVE_DEFRAG - if (server.active_defrag_enabled) { - server.active_defrag_enabled = 0; - addReplyError(c, - "-DISABLED Active defragmentation cannot be enabled: it " - "requires a Redis server compiled with a modified Jemalloc " - "like the one shipped by default with the Redis source " - "distribution"); - return; - } -#endif - } config_set_bool_field( - "protected-mode",server.protected_mode) { - } config_set_bool_field( - "stop-writes-on-bgsave-error",server.stop_writes_on_bgsave_err) { - } config_set_bool_field( - "lazyfree-lazy-eviction",server.lazyfree_lazy_eviction) { - } config_set_bool_field( - "lazyfree-lazy-expire",server.lazyfree_lazy_expire) { - } config_set_bool_field( - "lazyfree-lazy-server-del",server.lazyfree_lazy_server_del) { - } config_set_bool_field( - "slave-lazy-flush",server.repl_slave_lazy_flush) { - } config_set_bool_field( - "replica-lazy-flush",server.repl_slave_lazy_flush) { - } config_set_bool_field( - "no-appendfsync-on-rewrite",server.aof_no_fsync_on_rewrite) { - } config_set_bool_field( - "dynamic-hz",server.dynamic_hz) { - /* Numerical fields. * config_set_numerical_field(name,var,min,max) */ - } config_set_numerical_field( - "tcp-keepalive",server.tcpkeepalive,0,INT_MAX) { - } config_set_numerical_field( - "maxmemory-samples",server.maxmemory_samples,1,INT_MAX) { - } config_set_numerical_field( - "lfu-log-factor",server.lfu_log_factor,0,INT_MAX) { - } config_set_numerical_field( - "lfu-decay-time",server.lfu_decay_time,0,INT_MAX) { - } config_set_numerical_field( - "timeout",server.maxidletime,0,INT_MAX) { - } config_set_numerical_field( - "active-defrag-threshold-lower",server.active_defrag_threshold_lower,0,1000) { - } config_set_numerical_field( - "active-defrag-threshold-upper",server.active_defrag_threshold_upper,0,1000) { - } config_set_memory_field( - "active-defrag-ignore-bytes",server.active_defrag_ignore_bytes) { - } config_set_numerical_field( - "active-defrag-cycle-min",server.active_defrag_cycle_min,1,99) { - } config_set_numerical_field( - "active-defrag-cycle-max",server.active_defrag_cycle_max,1,99) { - } config_set_numerical_field( - "active-defrag-max-scan-fields",server.active_defrag_max_scan_fields,1,LONG_MAX) { - } config_set_numerical_field( - "auto-aof-rewrite-percentage",server.aof_rewrite_perc,0,INT_MAX){ - } config_set_numerical_field( - "hash-max-ziplist-entries",server.hash_max_ziplist_entries,0,LONG_MAX) { - } config_set_numerical_field( - "hash-max-ziplist-value",server.hash_max_ziplist_value,0,LONG_MAX) { - } config_set_numerical_field( - "stream-node-max-bytes",server.stream_node_max_bytes,0,LONG_MAX) { - } config_set_numerical_field( - "stream-node-max-entries",server.stream_node_max_entries,0,LLONG_MAX) { - } config_set_numerical_field( - "list-max-ziplist-size",server.list_max_ziplist_size,INT_MIN,INT_MAX) { - } config_set_numerical_field( - "list-compress-depth",server.list_compress_depth,0,INT_MAX) { - } config_set_numerical_field( - "set-max-intset-entries",server.set_max_intset_entries,0,LONG_MAX) { - } config_set_numerical_field( - "zset-max-ziplist-entries",server.zset_max_ziplist_entries,0,LONG_MAX) { - } config_set_numerical_field( - "zset-max-ziplist-value",server.zset_max_ziplist_value,0,LONG_MAX) { - } config_set_numerical_field( - "hll-sparse-max-bytes",server.hll_sparse_max_bytes,0,LONG_MAX) { - } config_set_numerical_field( - "lua-time-limit",server.lua_time_limit,0,LONG_MAX) { - } config_set_numerical_field( - "slowlog-log-slower-than",server.slowlog_log_slower_than,-1,LLONG_MAX) { - } config_set_numerical_field( - "slowlog-max-len",ll,0,LONG_MAX) { - /* Cast to unsigned. */ - server.slowlog_max_len = (unsigned long)ll; - } config_set_numerical_field( - "latency-monitor-threshold",server.latency_monitor_threshold,0,LLONG_MAX){ - } config_set_numerical_field( - "repl-ping-slave-period",server.repl_ping_slave_period,1,INT_MAX) { - } config_set_numerical_field( - "repl-ping-replica-period",server.repl_ping_slave_period,1,INT_MAX) { - } config_set_numerical_field( - "repl-timeout",server.repl_timeout,1,INT_MAX) { - } config_set_numerical_field( - "repl-backlog-ttl",server.repl_backlog_time_limit,0,LONG_MAX) { - } config_set_numerical_field( - "repl-diskless-sync-delay",server.repl_diskless_sync_delay,0,INT_MAX) { - } config_set_numerical_field( - "slave-priority",server.slave_priority,0,INT_MAX) { - } config_set_numerical_field( - "replica-priority",server.slave_priority,0,INT_MAX) { - } config_set_numerical_field( - "slave-announce-port",server.slave_announce_port,0,65535) { - } config_set_numerical_field( - "replica-announce-port",server.slave_announce_port,0,65535) { - } config_set_numerical_field( - "min-slaves-to-write",server.repl_min_slaves_to_write,0,INT_MAX) { - refreshGoodSlavesCount(); - } config_set_numerical_field( - "min-replicas-to-write",server.repl_min_slaves_to_write,0,INT_MAX) { - refreshGoodSlavesCount(); - } config_set_numerical_field( - "min-slaves-max-lag",server.repl_min_slaves_max_lag,0,INT_MAX) { - refreshGoodSlavesCount(); - } config_set_numerical_field( - "min-replicas-max-lag",server.repl_min_slaves_max_lag,0,INT_MAX) { - refreshGoodSlavesCount(); - } config_set_numerical_field( - "cluster-node-timeout",server.cluster_node_timeout,0,LLONG_MAX) { - } config_set_numerical_field( - "cluster-announce-port",server.cluster_announce_port,0,65535) { - } config_set_numerical_field( - "cluster-announce-bus-port",server.cluster_announce_bus_port,0,65535) { - } config_set_numerical_field( - "cluster-migration-barrier",server.cluster_migration_barrier,0,INT_MAX){ - } config_set_numerical_field( - "cluster-slave-validity-factor",server.cluster_slave_validity_factor,0,INT_MAX) { - } config_set_numerical_field( - "cluster-replica-validity-factor",server.cluster_slave_validity_factor,0,INT_MAX) { - } config_set_numerical_field( - "hz",server.config_hz,0,INT_MAX) { - /* Hz is more an hint from the user, so we accept values out of range - * but cap them to reasonable values. */ - if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; - if (server.config_hz > CONFIG_MAX_HZ) server.config_hz = CONFIG_MAX_HZ; } config_set_numerical_field( "watchdog-period",ll,0,INT_MAX) { if (ll) enableWatchdog(ll); else disableWatchdog(); - /* Memory fields. * config_set_memory_field(name,var) */ - } config_set_memory_field("maxmemory",server.maxmemory) { - if (server.maxmemory) { - if (server.maxmemory < zmalloc_used_memory()) { - serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy."); - } - freeMemoryIfNeededAndSafe(); - } - } config_set_memory_field( - "proto-max-bulk-len",server.proto_max_bulk_len) { } config_set_memory_field( "client-query-buffer-limit",server.client_max_querybuf_len) { - } config_set_memory_field("repl-backlog-size",ll) { - resizeReplicationBacklog(ll); - } config_set_memory_field("auto-aof-rewrite-min-size",ll) { - server.aof_rewrite_min_size = ll; - - /* Enumeration fields. - * config_set_enum_field(name,var,enum_var) */ - } config_set_enum_field( - "loglevel",server.verbosity,loglevel_enum) { - } config_set_enum_field( - "maxmemory-policy",server.maxmemory_policy,maxmemory_policy_enum) { - } config_set_enum_field( - "appendfsync",server.aof_fsync,aof_fsync_enum) { - - /* Everyhing else is an error... */ + /* Everything else is an error... */ } config_set_else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", (char*)c->argv[2]->ptr); @@ -1280,9 +750,16 @@ void configSetCommand(client *c) { return; badfmt: /* Bad format errors */ - addReplyErrorFormat(c,"Invalid argument '%s' for CONFIG SET '%s'", - (char*)o->ptr, - (char*)c->argv[2]->ptr); + if (errstr) { + addReplyErrorFormat(c,"Invalid argument '%s' for CONFIG SET '%s' - %s", + (char*)o->ptr, + (char*)c->argv[2]->ptr, + errstr); + } else { + addReplyErrorFormat(c,"Invalid argument '%s' for CONFIG SET '%s'", + (char*)o->ptr, + (char*)c->argv[2]->ptr); + } } /*----------------------------------------------------------------------------- @@ -1314,179 +791,38 @@ void configSetCommand(client *c) { } \ } while(0); -#define config_get_enum_field(_name,_var,_enumvar) do { \ - if (stringmatch(pattern,_name,1)) { \ - addReplyBulkCString(c,_name); \ - addReplyBulkCString(c,configEnumGetNameOrUnknown(_enumvar,_var)); \ - matches++; \ - } \ -} while(0); void configGetCommand(client *c) { robj *o = c->argv[2]; - void *replylen = addDeferredMultiBulkLength(c); + void *replylen = addReplyDeferredLen(c); char *pattern = o->ptr; char buf[128]; int matches = 0; serverAssertWithInfo(c,o,sdsEncodedObject(o)); + /* Iterate the configs that are standard */ + for (standardConfig *config = configs; config->name != NULL; config++) { + if (stringmatch(pattern,config->name,1)) { + addReplyBulkCString(c,config->name); + config->interface.get(c,config->data); + matches++; + } + if (config->alias && stringmatch(pattern,config->alias,1)) { + addReplyBulkCString(c,config->alias); + config->interface.get(c,config->data); + matches++; + } + } + /* String values */ - config_get_string_field("dbfilename",server.rdb_filename); - config_get_string_field("requirepass",server.requirepass); - config_get_string_field("masterauth",server.masterauth); - config_get_string_field("cluster-announce-ip",server.cluster_announce_ip); - config_get_string_field("unixsocket",server.unixsocket); config_get_string_field("logfile",server.logfile); - config_get_string_field("pidfile",server.pidfile); - config_get_string_field("slave-announce-ip",server.slave_announce_ip); - config_get_string_field("replica-announce-ip",server.slave_announce_ip); /* Numerical values */ - config_get_numerical_field("maxmemory",server.maxmemory); - config_get_numerical_field("proto-max-bulk-len",server.proto_max_bulk_len); config_get_numerical_field("client-query-buffer-limit",server.client_max_querybuf_len); - config_get_numerical_field("maxmemory-samples",server.maxmemory_samples); - config_get_numerical_field("lfu-log-factor",server.lfu_log_factor); - config_get_numerical_field("lfu-decay-time",server.lfu_decay_time); - config_get_numerical_field("timeout",server.maxidletime); - config_get_numerical_field("active-defrag-threshold-lower",server.active_defrag_threshold_lower); - config_get_numerical_field("active-defrag-threshold-upper",server.active_defrag_threshold_upper); - config_get_numerical_field("active-defrag-ignore-bytes",server.active_defrag_ignore_bytes); - config_get_numerical_field("active-defrag-cycle-min",server.active_defrag_cycle_min); - config_get_numerical_field("active-defrag-cycle-max",server.active_defrag_cycle_max); - config_get_numerical_field("active-defrag-max-scan-fields",server.active_defrag_max_scan_fields); - config_get_numerical_field("auto-aof-rewrite-percentage", - server.aof_rewrite_perc); - config_get_numerical_field("auto-aof-rewrite-min-size", - server.aof_rewrite_min_size); - config_get_numerical_field("hash-max-ziplist-entries", - server.hash_max_ziplist_entries); - config_get_numerical_field("hash-max-ziplist-value", - server.hash_max_ziplist_value); - config_get_numerical_field("stream-node-max-bytes", - server.stream_node_max_bytes); - config_get_numerical_field("stream-node-max-entries", - server.stream_node_max_entries); - config_get_numerical_field("list-max-ziplist-size", - server.list_max_ziplist_size); - config_get_numerical_field("list-compress-depth", - server.list_compress_depth); - config_get_numerical_field("set-max-intset-entries", - server.set_max_intset_entries); - config_get_numerical_field("zset-max-ziplist-entries", - server.zset_max_ziplist_entries); - config_get_numerical_field("zset-max-ziplist-value", - server.zset_max_ziplist_value); - config_get_numerical_field("hll-sparse-max-bytes", - server.hll_sparse_max_bytes); - config_get_numerical_field("lua-time-limit",server.lua_time_limit); - config_get_numerical_field("slowlog-log-slower-than", - server.slowlog_log_slower_than); - config_get_numerical_field("latency-monitor-threshold", - server.latency_monitor_threshold); - config_get_numerical_field("slowlog-max-len", - server.slowlog_max_len); - config_get_numerical_field("port",server.port); - config_get_numerical_field("cluster-announce-port",server.cluster_announce_port); - config_get_numerical_field("cluster-announce-bus-port",server.cluster_announce_bus_port); - config_get_numerical_field("tcp-backlog",server.tcp_backlog); - config_get_numerical_field("databases",server.dbnum); - config_get_numerical_field("repl-ping-slave-period",server.repl_ping_slave_period); - config_get_numerical_field("repl-ping-replica-period",server.repl_ping_slave_period); - config_get_numerical_field("repl-timeout",server.repl_timeout); - config_get_numerical_field("repl-backlog-size",server.repl_backlog_size); - config_get_numerical_field("repl-backlog-ttl",server.repl_backlog_time_limit); - config_get_numerical_field("maxclients",server.maxclients); config_get_numerical_field("watchdog-period",server.watchdog_period); - config_get_numerical_field("slave-priority",server.slave_priority); - config_get_numerical_field("replica-priority",server.slave_priority); - config_get_numerical_field("slave-announce-port",server.slave_announce_port); - config_get_numerical_field("replica-announce-port",server.slave_announce_port); - config_get_numerical_field("min-slaves-to-write",server.repl_min_slaves_to_write); - config_get_numerical_field("min-replicas-to-write",server.repl_min_slaves_to_write); - config_get_numerical_field("min-slaves-max-lag",server.repl_min_slaves_max_lag); - config_get_numerical_field("min-replicas-max-lag",server.repl_min_slaves_max_lag); - config_get_numerical_field("hz",server.config_hz); - config_get_numerical_field("cluster-node-timeout",server.cluster_node_timeout); - config_get_numerical_field("cluster-migration-barrier",server.cluster_migration_barrier); - config_get_numerical_field("cluster-slave-validity-factor",server.cluster_slave_validity_factor); - config_get_numerical_field("cluster-replica-validity-factor",server.cluster_slave_validity_factor); - config_get_numerical_field("repl-diskless-sync-delay",server.repl_diskless_sync_delay); - config_get_numerical_field("tcp-keepalive",server.tcpkeepalive); - - /* Bool (yes/no) values */ - config_get_bool_field("cluster-require-full-coverage", - server.cluster_require_full_coverage); - config_get_bool_field("cluster-slave-no-failover", - server.cluster_slave_no_failover); - config_get_bool_field("cluster-replica-no-failover", - server.cluster_slave_no_failover); - config_get_bool_field("no-appendfsync-on-rewrite", - server.aof_no_fsync_on_rewrite); - config_get_bool_field("slave-serve-stale-data", - server.repl_serve_stale_data); - config_get_bool_field("replica-serve-stale-data", - server.repl_serve_stale_data); - config_get_bool_field("slave-read-only", - server.repl_slave_ro); - config_get_bool_field("replica-read-only", - server.repl_slave_ro); - config_get_bool_field("slave-ignore-maxmemory", - server.repl_slave_ignore_maxmemory); - config_get_bool_field("replica-ignore-maxmemory", - server.repl_slave_ignore_maxmemory); - config_get_bool_field("stop-writes-on-bgsave-error", - server.stop_writes_on_bgsave_err); - config_get_bool_field("daemonize", server.daemonize); - config_get_bool_field("rdbcompression", server.rdb_compression); - config_get_bool_field("rdbchecksum", server.rdb_checksum); - config_get_bool_field("activerehashing", server.activerehashing); - config_get_bool_field("activedefrag", server.active_defrag_enabled); - config_get_bool_field("protected-mode", server.protected_mode); - config_get_bool_field("repl-disable-tcp-nodelay", - server.repl_disable_tcp_nodelay); - config_get_bool_field("repl-diskless-sync", - server.repl_diskless_sync); - config_get_bool_field("aof-rewrite-incremental-fsync", - server.aof_rewrite_incremental_fsync); - config_get_bool_field("rdb-save-incremental-fsync", - server.rdb_save_incremental_fsync); - config_get_bool_field("aof-load-truncated", - server.aof_load_truncated); - config_get_bool_field("aof-use-rdb-preamble", - server.aof_use_rdb_preamble); - config_get_bool_field("lazyfree-lazy-eviction", - server.lazyfree_lazy_eviction); - config_get_bool_field("lazyfree-lazy-expire", - server.lazyfree_lazy_expire); - config_get_bool_field("lazyfree-lazy-server-del", - server.lazyfree_lazy_server_del); - config_get_bool_field("slave-lazy-flush", - server.repl_slave_lazy_flush); - config_get_bool_field("replica-lazy-flush", - server.repl_slave_lazy_flush); - config_get_bool_field("dynamic-hz", - server.dynamic_hz); - - /* Enum values */ - config_get_enum_field("maxmemory-policy", - server.maxmemory_policy,maxmemory_policy_enum); - config_get_enum_field("loglevel", - server.verbosity,loglevel_enum); - config_get_enum_field("supervised", - server.supervised_mode,supervised_mode_enum); - config_get_enum_field("appendfsync", - server.aof_fsync,aof_fsync_enum); - config_get_enum_field("syslog-facility", - server.syslog_facility,syslog_facility_enum); /* Everything we can't handle with macros follows. */ - if (stringmatch(pattern,"appendonly",1)) { - addReplyBulkCString(c,"appendonly"); - addReplyBulkCString(c,server.aof_state == AOF_OFF ? "no" : "yes"); - matches++; - } if (stringmatch(pattern,"dir",1)) { char buf[1024]; @@ -1555,12 +891,10 @@ void configGetCommand(client *c) { matches++; } if (stringmatch(pattern,"notify-keyspace-events",1)) { - robj *flagsobj = createObject(OBJ_STRING, - keyspaceEventsFlagsToString(server.notify_keyspace_events)); + sds flags = keyspaceEventsFlagsToString(server.notify_keyspace_events); addReplyBulkCString(c,"notify-keyspace-events"); - addReplyBulk(c,flagsobj); - decrRefCount(flagsobj); + addReplyBulkSds(c,flags); matches++; } if (stringmatch(pattern,"bind",1)) { @@ -1571,7 +905,18 @@ void configGetCommand(client *c) { sdsfree(aux); matches++; } - setDeferredMultiBulkLength(c,replylen,matches*2); + if (stringmatch(pattern,"requirepass",1)) { + addReplyBulkCString(c,"requirepass"); + sds password = server.requirepass; + if (password) { + addReplyBulkCBuffer(c,password,sdslen(password)); + } else { + addReplyBulkCString(c,""); + } + matches++; + } + + setDeferredMapLen(c,replylen,matches); } /*----------------------------------------------------------------------------- @@ -1790,7 +1135,7 @@ int rewriteConfigFormatMemory(char *buf, size_t len, long long bytes) { } /* Rewrite a simple "option-name " configuration option. */ -void rewriteConfigBytesOption(struct rewriteConfigState *state, char *option, long long value, long long defvalue) { +void rewriteConfigBytesOption(struct rewriteConfigState *state, const char *option, long long value, long long defvalue) { char buf[64]; int force = value != defvalue; sds line; @@ -1801,7 +1146,7 @@ void rewriteConfigBytesOption(struct rewriteConfigState *state, char *option, lo } /* Rewrite a yes/no option. */ -void rewriteConfigYesNoOption(struct rewriteConfigState *state, char *option, int value, int defvalue) { +void rewriteConfigYesNoOption(struct rewriteConfigState *state, const char *option, int value, int defvalue) { int force = value != defvalue; sds line = sdscatprintf(sdsempty(),"%s %s",option, value ? "yes" : "no"); @@ -1810,7 +1155,7 @@ void rewriteConfigYesNoOption(struct rewriteConfigState *state, char *option, in } /* Rewrite a string option. */ -void rewriteConfigStringOption(struct rewriteConfigState *state, char *option, char *value, char *defvalue) { +void rewriteConfigStringOption(struct rewriteConfigState *state, const char *option, char *value, const char *defvalue) { int force = 1; sds line; @@ -1832,7 +1177,7 @@ void rewriteConfigStringOption(struct rewriteConfigState *state, char *option, c } /* Rewrite a numerical (long long range) option. */ -void rewriteConfigNumericalOption(struct rewriteConfigState *state, char *option, long long value, long long defvalue) { +void rewriteConfigNumericalOption(struct rewriteConfigState *state, const char *option, long long value, long long defvalue) { int force = value != defvalue; sds line = sdscatprintf(sdsempty(),"%s %lld",option,value); @@ -1850,7 +1195,7 @@ void rewriteConfigOctalOption(struct rewriteConfigState *state, char *option, in /* Rewrite an enumeration option. It takes as usually state and option name, * and in addition the enumeration array and the default value for the * option. */ -void rewriteConfigEnumOption(struct rewriteConfigState *state, char *option, int value, configEnum *ce, int defval) { +void rewriteConfigEnumOption(struct rewriteConfigState *state, const char *option, int value, configEnum *ce, int defval) { sds line; const char *name = configEnumGetNameOrUnknown(ce,value); int force = value != defval; @@ -1859,18 +1204,6 @@ void rewriteConfigEnumOption(struct rewriteConfigState *state, char *option, int rewriteConfigRewriteLine(state,option,line,force); } -/* Rewrite the syslog-facility option. */ -void rewriteConfigSyslogfacilityOption(struct rewriteConfigState *state) { - int value = server.syslog_facility; - int force = value != LOG_LOCAL0; - const char *name = NULL, *option = "syslog-facility"; - sds line; - - name = configEnumGetNameOrUnknown(syslog_facility_enum,value); - line = sdscatprintf(sdsempty(),"%s %s",option,name); - rewriteConfigRewriteLine(state,option,line,force); -} - /* Rewrite the save option. */ void rewriteConfigSaveOption(struct rewriteConfigState *state) { int j; @@ -1888,6 +1221,38 @@ void rewriteConfigSaveOption(struct rewriteConfigState *state) { rewriteConfigMarkAsProcessed(state,"save"); } +/* Rewrite the user option. */ +void rewriteConfigUserOption(struct rewriteConfigState *state) { + /* If there is a user file defined we just mark this configuration + * directive as processed, so that all the lines containing users + * inside the config file gets discarded. */ + if (server.acl_filename[0] != '\0') { + rewriteConfigMarkAsProcessed(state,"user"); + return; + } + + /* Otherwise scan the list of users and rewrite every line. Note that + * in case the list here is empty, the effect will just be to comment + * all the users directive inside the config file. */ + raxIterator ri; + raxStart(&ri,Users); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + user *u = ri.data; + sds line = sdsnew("user "); + line = sdscatsds(line,u->name); + line = sdscatlen(line," ",1); + sds descr = ACLDescribeUser(u); + line = sdscatsds(line,descr); + sdsfree(descr); + rewriteConfigRewriteLine(state,"user",line,1); + } + raxStop(&ri); + + /* Mark "user" as processed in case there are no defined users. */ + rewriteConfigMarkAsProcessed(state,"user"); +} + /* Rewrite the dir option, always using absolute paths.*/ void rewriteConfigDirOption(struct rewriteConfigState *state) { char cwd[1024]; @@ -1980,6 +1345,26 @@ void rewriteConfigBindOption(struct rewriteConfigState *state) { rewriteConfigRewriteLine(state,option,line,force); } +/* Rewrite the requirepass option. */ +void rewriteConfigRequirepassOption(struct rewriteConfigState *state, char *option) { + int force = 1; + sds line; + sds password = server.requirepass; + + /* If there is no password set, we don't want the requirepass option + * to be present in the configuration at all. */ + if (password == NULL) { + rewriteConfigMarkAsProcessed(state,option); + return; + } + + line = sdsnew(option); + line = sdscatlen(line, " ", 1); + line = sdscatsds(line, password); + + rewriteConfigRewriteLine(state,option,line,force); +} + /* Glue together the configuration lines in the current configuration * rewrite state into a single string, stripping multiple empty lines. */ sds rewriteConfigGetContentFromState(struct rewriteConfigState *state) { @@ -2119,105 +1504,23 @@ int rewriteConfig(char *path) { /* Step 2: rewrite every single option, replacing or appending it inside * the rewrite state. */ - rewriteConfigYesNoOption(state,"daemonize",server.daemonize,0); - rewriteConfigStringOption(state,"pidfile",server.pidfile,CONFIG_DEFAULT_PID_FILE); - rewriteConfigNumericalOption(state,"port",server.port,CONFIG_DEFAULT_SERVER_PORT); - rewriteConfigNumericalOption(state,"cluster-announce-port",server.cluster_announce_port,CONFIG_DEFAULT_CLUSTER_ANNOUNCE_PORT); - rewriteConfigNumericalOption(state,"cluster-announce-bus-port",server.cluster_announce_bus_port,CONFIG_DEFAULT_CLUSTER_ANNOUNCE_BUS_PORT); - rewriteConfigNumericalOption(state,"tcp-backlog",server.tcp_backlog,CONFIG_DEFAULT_TCP_BACKLOG); + /* Iterate the configs that are standard */ + for (standardConfig *config = configs; config->name != NULL; config++) { + config->interface.rewrite(config->data, config->name, state); + } + rewriteConfigBindOption(state); - rewriteConfigStringOption(state,"unixsocket",server.unixsocket,NULL); rewriteConfigOctalOption(state,"unixsocketperm",server.unixsocketperm,CONFIG_DEFAULT_UNIX_SOCKET_PERM); - rewriteConfigNumericalOption(state,"timeout",server.maxidletime,CONFIG_DEFAULT_CLIENT_TIMEOUT); - rewriteConfigNumericalOption(state,"tcp-keepalive",server.tcpkeepalive,CONFIG_DEFAULT_TCP_KEEPALIVE); - rewriteConfigNumericalOption(state,"replica-announce-port",server.slave_announce_port,CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT); - rewriteConfigEnumOption(state,"loglevel",server.verbosity,loglevel_enum,CONFIG_DEFAULT_VERBOSITY); rewriteConfigStringOption(state,"logfile",server.logfile,CONFIG_DEFAULT_LOGFILE); - rewriteConfigYesNoOption(state,"syslog-enabled",server.syslog_enabled,CONFIG_DEFAULT_SYSLOG_ENABLED); - rewriteConfigStringOption(state,"syslog-ident",server.syslog_ident,CONFIG_DEFAULT_SYSLOG_IDENT); - rewriteConfigSyslogfacilityOption(state); rewriteConfigSaveOption(state); - rewriteConfigNumericalOption(state,"databases",server.dbnum,CONFIG_DEFAULT_DBNUM); - rewriteConfigYesNoOption(state,"stop-writes-on-bgsave-error",server.stop_writes_on_bgsave_err,CONFIG_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR); - rewriteConfigYesNoOption(state,"rdbcompression",server.rdb_compression,CONFIG_DEFAULT_RDB_COMPRESSION); - rewriteConfigYesNoOption(state,"rdbchecksum",server.rdb_checksum,CONFIG_DEFAULT_RDB_CHECKSUM); - rewriteConfigStringOption(state,"dbfilename",server.rdb_filename,CONFIG_DEFAULT_RDB_FILENAME); + rewriteConfigUserOption(state); rewriteConfigDirOption(state); rewriteConfigSlaveofOption(state,"replicaof"); - rewriteConfigStringOption(state,"replica-announce-ip",server.slave_announce_ip,CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP); - rewriteConfigStringOption(state,"masterauth",server.masterauth,NULL); - rewriteConfigStringOption(state,"cluster-announce-ip",server.cluster_announce_ip,NULL); - rewriteConfigYesNoOption(state,"replica-serve-stale-data",server.repl_serve_stale_data,CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA); - rewriteConfigYesNoOption(state,"replica-read-only",server.repl_slave_ro,CONFIG_DEFAULT_SLAVE_READ_ONLY); - rewriteConfigYesNoOption(state,"replica-ignore-maxmemory",server.repl_slave_ignore_maxmemory,CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY); - rewriteConfigNumericalOption(state,"repl-ping-replica-period",server.repl_ping_slave_period,CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD); - rewriteConfigNumericalOption(state,"repl-timeout",server.repl_timeout,CONFIG_DEFAULT_REPL_TIMEOUT); - rewriteConfigBytesOption(state,"repl-backlog-size",server.repl_backlog_size,CONFIG_DEFAULT_REPL_BACKLOG_SIZE); - rewriteConfigBytesOption(state,"repl-backlog-ttl",server.repl_backlog_time_limit,CONFIG_DEFAULT_REPL_BACKLOG_TIME_LIMIT); - rewriteConfigYesNoOption(state,"repl-disable-tcp-nodelay",server.repl_disable_tcp_nodelay,CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY); - rewriteConfigYesNoOption(state,"repl-diskless-sync",server.repl_diskless_sync,CONFIG_DEFAULT_REPL_DISKLESS_SYNC); - rewriteConfigNumericalOption(state,"repl-diskless-sync-delay",server.repl_diskless_sync_delay,CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY); - rewriteConfigNumericalOption(state,"replica-priority",server.slave_priority,CONFIG_DEFAULT_SLAVE_PRIORITY); - rewriteConfigNumericalOption(state,"min-replicas-to-write",server.repl_min_slaves_to_write,CONFIG_DEFAULT_MIN_SLAVES_TO_WRITE); - rewriteConfigNumericalOption(state,"min-replicas-max-lag",server.repl_min_slaves_max_lag,CONFIG_DEFAULT_MIN_SLAVES_MAX_LAG); - rewriteConfigStringOption(state,"requirepass",server.requirepass,NULL); - rewriteConfigNumericalOption(state,"maxclients",server.maxclients,CONFIG_DEFAULT_MAX_CLIENTS); - rewriteConfigBytesOption(state,"maxmemory",server.maxmemory,CONFIG_DEFAULT_MAXMEMORY); - rewriteConfigBytesOption(state,"proto-max-bulk-len",server.proto_max_bulk_len,CONFIG_DEFAULT_PROTO_MAX_BULK_LEN); + rewriteConfigRequirepassOption(state,"requirepass"); rewriteConfigBytesOption(state,"client-query-buffer-limit",server.client_max_querybuf_len,PROTO_MAX_QUERYBUF_LEN); - rewriteConfigEnumOption(state,"maxmemory-policy",server.maxmemory_policy,maxmemory_policy_enum,CONFIG_DEFAULT_MAXMEMORY_POLICY); - rewriteConfigNumericalOption(state,"maxmemory-samples",server.maxmemory_samples,CONFIG_DEFAULT_MAXMEMORY_SAMPLES); - rewriteConfigNumericalOption(state,"lfu-log-factor",server.lfu_log_factor,CONFIG_DEFAULT_LFU_LOG_FACTOR); - rewriteConfigNumericalOption(state,"lfu-decay-time",server.lfu_decay_time,CONFIG_DEFAULT_LFU_DECAY_TIME); - rewriteConfigNumericalOption(state,"active-defrag-threshold-lower",server.active_defrag_threshold_lower,CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER); - rewriteConfigNumericalOption(state,"active-defrag-threshold-upper",server.active_defrag_threshold_upper,CONFIG_DEFAULT_DEFRAG_THRESHOLD_UPPER); - rewriteConfigBytesOption(state,"active-defrag-ignore-bytes",server.active_defrag_ignore_bytes,CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES); - rewriteConfigNumericalOption(state,"active-defrag-cycle-min",server.active_defrag_cycle_min,CONFIG_DEFAULT_DEFRAG_CYCLE_MIN); - rewriteConfigNumericalOption(state,"active-defrag-cycle-max",server.active_defrag_cycle_max,CONFIG_DEFAULT_DEFRAG_CYCLE_MAX); - rewriteConfigNumericalOption(state,"active-defrag-max-scan-fields",server.active_defrag_max_scan_fields,CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS); - rewriteConfigYesNoOption(state,"appendonly",server.aof_state != AOF_OFF,0); - rewriteConfigStringOption(state,"appendfilename",server.aof_filename,CONFIG_DEFAULT_AOF_FILENAME); - rewriteConfigEnumOption(state,"appendfsync",server.aof_fsync,aof_fsync_enum,CONFIG_DEFAULT_AOF_FSYNC); - rewriteConfigYesNoOption(state,"no-appendfsync-on-rewrite",server.aof_no_fsync_on_rewrite,CONFIG_DEFAULT_AOF_NO_FSYNC_ON_REWRITE); - rewriteConfigNumericalOption(state,"auto-aof-rewrite-percentage",server.aof_rewrite_perc,AOF_REWRITE_PERC); - rewriteConfigBytesOption(state,"auto-aof-rewrite-min-size",server.aof_rewrite_min_size,AOF_REWRITE_MIN_SIZE); - rewriteConfigNumericalOption(state,"lua-time-limit",server.lua_time_limit,LUA_SCRIPT_TIME_LIMIT); - rewriteConfigYesNoOption(state,"cluster-enabled",server.cluster_enabled,0); rewriteConfigStringOption(state,"cluster-config-file",server.cluster_configfile,CONFIG_DEFAULT_CLUSTER_CONFIG_FILE); - rewriteConfigYesNoOption(state,"cluster-require-full-coverage",server.cluster_require_full_coverage,CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE); - rewriteConfigYesNoOption(state,"cluster-replica-no-failover",server.cluster_slave_no_failover,CLUSTER_DEFAULT_SLAVE_NO_FAILOVER); - rewriteConfigNumericalOption(state,"cluster-node-timeout",server.cluster_node_timeout,CLUSTER_DEFAULT_NODE_TIMEOUT); - rewriteConfigNumericalOption(state,"cluster-migration-barrier",server.cluster_migration_barrier,CLUSTER_DEFAULT_MIGRATION_BARRIER); - rewriteConfigNumericalOption(state,"cluster-replica-validity-factor",server.cluster_slave_validity_factor,CLUSTER_DEFAULT_SLAVE_VALIDITY); - rewriteConfigNumericalOption(state,"slowlog-log-slower-than",server.slowlog_log_slower_than,CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN); - rewriteConfigNumericalOption(state,"latency-monitor-threshold",server.latency_monitor_threshold,CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD); - rewriteConfigNumericalOption(state,"slowlog-max-len",server.slowlog_max_len,CONFIG_DEFAULT_SLOWLOG_MAX_LEN); rewriteConfigNotifykeyspaceeventsOption(state); - rewriteConfigNumericalOption(state,"hash-max-ziplist-entries",server.hash_max_ziplist_entries,OBJ_HASH_MAX_ZIPLIST_ENTRIES); - rewriteConfigNumericalOption(state,"hash-max-ziplist-value",server.hash_max_ziplist_value,OBJ_HASH_MAX_ZIPLIST_VALUE); - rewriteConfigNumericalOption(state,"stream-node-max-bytes",server.stream_node_max_bytes,OBJ_STREAM_NODE_MAX_BYTES); - rewriteConfigNumericalOption(state,"stream-node-max-entries",server.stream_node_max_entries,OBJ_STREAM_NODE_MAX_ENTRIES); - rewriteConfigNumericalOption(state,"list-max-ziplist-size",server.list_max_ziplist_size,OBJ_LIST_MAX_ZIPLIST_SIZE); - rewriteConfigNumericalOption(state,"list-compress-depth",server.list_compress_depth,OBJ_LIST_COMPRESS_DEPTH); - rewriteConfigNumericalOption(state,"set-max-intset-entries",server.set_max_intset_entries,OBJ_SET_MAX_INTSET_ENTRIES); - rewriteConfigNumericalOption(state,"zset-max-ziplist-entries",server.zset_max_ziplist_entries,OBJ_ZSET_MAX_ZIPLIST_ENTRIES); - rewriteConfigNumericalOption(state,"zset-max-ziplist-value",server.zset_max_ziplist_value,OBJ_ZSET_MAX_ZIPLIST_VALUE); - rewriteConfigNumericalOption(state,"hll-sparse-max-bytes",server.hll_sparse_max_bytes,CONFIG_DEFAULT_HLL_SPARSE_MAX_BYTES); - rewriteConfigYesNoOption(state,"activerehashing",server.activerehashing,CONFIG_DEFAULT_ACTIVE_REHASHING); - rewriteConfigYesNoOption(state,"activedefrag",server.active_defrag_enabled,CONFIG_DEFAULT_ACTIVE_DEFRAG); - rewriteConfigYesNoOption(state,"protected-mode",server.protected_mode,CONFIG_DEFAULT_PROTECTED_MODE); rewriteConfigClientoutputbufferlimitOption(state); - rewriteConfigNumericalOption(state,"hz",server.config_hz,CONFIG_DEFAULT_HZ); - rewriteConfigYesNoOption(state,"aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync,CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC); - rewriteConfigYesNoOption(state,"rdb-save-incremental-fsync",server.rdb_save_incremental_fsync,CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC); - rewriteConfigYesNoOption(state,"aof-load-truncated",server.aof_load_truncated,CONFIG_DEFAULT_AOF_LOAD_TRUNCATED); - rewriteConfigYesNoOption(state,"aof-use-rdb-preamble",server.aof_use_rdb_preamble,CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE); - rewriteConfigEnumOption(state,"supervised",server.supervised_mode,supervised_mode_enum,SUPERVISED_NONE); - rewriteConfigYesNoOption(state,"lazyfree-lazy-eviction",server.lazyfree_lazy_eviction,CONFIG_DEFAULT_LAZYFREE_LAZY_EVICTION); - rewriteConfigYesNoOption(state,"lazyfree-lazy-expire",server.lazyfree_lazy_expire,CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE); - rewriteConfigYesNoOption(state,"lazyfree-lazy-server-del",server.lazyfree_lazy_server_del,CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL); - rewriteConfigYesNoOption(state,"replica-lazy-flush",server.repl_slave_lazy_flush,CONFIG_DEFAULT_SLAVE_LAZY_FLUSH); - rewriteConfigYesNoOption(state,"dynamic-hz",server.dynamic_hz,CONFIG_DEFAULT_DYNAMIC_HZ); /* Rewrite Sentinel config if in Sentinel mode. */ if (server.sentinel_mode) rewriteConfigSentinelOption(state); @@ -2237,6 +1540,700 @@ int rewriteConfig(char *path) { return retval; } +/*----------------------------------------------------------------------------- + * Configs that fit one of the major types and require no special handling + *----------------------------------------------------------------------------*/ +#define LOADBUF_SIZE 256 +static char loadbuf[LOADBUF_SIZE]; + +#define MODIFIABLE_CONFIG 1 +#define IMMUTABLE_CONFIG 0 + +#define embedCommonConfig(config_name, config_alias, is_modifiable) \ + .name = (config_name), \ + .alias = (config_alias), \ + .modifiable = (is_modifiable), + +#define embedConfigInterface(initfn, setfn, getfn, rewritefn) .interface = { \ + .init = (initfn), \ + .set = (setfn), \ + .get = (getfn), \ + .rewrite = (rewritefn) \ +}, + +/* What follows is the generic config types that are supported. To add a new + * config with one of these types, add it to the standardConfig table with + * the creation macro for each type. + * + * Each type contains the following: + * * A function defining how to load this type on startup. + * * A function defining how to update this type on CONFIG SET. + * * A function defining how to serialize this type on CONFIG SET. + * * A function defining how to rewrite this type on CONFIG REWRITE. + * * A Macro defining how to create this type. + */ + +/* Bool Configs */ +static void boolConfigInit(typeData data) { + *data.yesno.config = data.yesno.default_value; +} + +static int boolConfigSet(typeData data, sds value, int update, char **err) { + int yn = yesnotoi(value); + if (yn == -1) { + *err = "argument must be 'yes' or 'no'"; + return 0; + } + if (data.yesno.is_valid_fn && !data.yesno.is_valid_fn(yn, err)) + return 0; + int prev = *(data.yesno.config); + *(data.yesno.config) = yn; + if (update && data.yesno.update_fn && !data.yesno.update_fn(yn, prev, err)) { + *(data.yesno.config) = prev; + return 0; + } + return 1; +} + +static void boolConfigGet(client *c, typeData data) { + addReplyBulkCString(c, *data.yesno.config ? "yes" : "no"); +} + +static void boolConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + rewriteConfigYesNoOption(state, name,*(data.yesno.config), data.yesno.default_value); +} + +#define createBoolConfig(name, alias, modifiable, config_addr, default, is_valid, update) { \ + embedCommonConfig(name, alias, modifiable) \ + embedConfigInterface(boolConfigInit, boolConfigSet, boolConfigGet, boolConfigRewrite) \ + .data.yesno = { \ + .config = &(config_addr), \ + .default_value = (default), \ + .is_valid_fn = (is_valid), \ + .update_fn = (update), \ + } \ +} + +/* String Configs */ +static void stringConfigInit(typeData data) { + if (data.string.convert_empty_to_null) { + *data.string.config = data.string.default_value ? zstrdup(data.string.default_value) : NULL; + } else { + *data.string.config = zstrdup(data.string.default_value); + } +} + +static int stringConfigSet(typeData data, sds value, int update, char **err) { + if (data.string.is_valid_fn && !data.string.is_valid_fn(value, err)) + return 0; + char *prev = *data.string.config; + if (data.string.convert_empty_to_null) { + *data.string.config = value[0] ? zstrdup(value) : NULL; + } else { + *data.string.config = zstrdup(value); + } + if (update && data.string.update_fn && !data.string.update_fn(*data.string.config, prev, err)) { + zfree(*data.string.config); + *data.string.config = prev; + return 0; + } + zfree(prev); + return 1; +} + +static void stringConfigGet(client *c, typeData data) { + addReplyBulkCString(c, *data.string.config ? *data.string.config : ""); +} + +static void stringConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + rewriteConfigStringOption(state, name,*(data.string.config), data.string.default_value); +} + +#define ALLOW_EMPTY_STRING 0 +#define EMPTY_STRING_IS_NULL 1 + +#define createStringConfig(name, alias, modifiable, empty_to_null, config_addr, default, is_valid, update) { \ + embedCommonConfig(name, alias, modifiable) \ + embedConfigInterface(stringConfigInit, stringConfigSet, stringConfigGet, stringConfigRewrite) \ + .data.string = { \ + .config = &(config_addr), \ + .default_value = (default), \ + .is_valid_fn = (is_valid), \ + .update_fn = (update), \ + .convert_empty_to_null = (empty_to_null), \ + } \ +} + +/* Enum configs */ +static void enumConfigInit(typeData data) { + *data.enumd.config = data.enumd.default_value; +} + +static int enumConfigSet(typeData data, sds value, int update, char **err) { + int enumval = configEnumGetValue(data.enumd.enum_value, value); + if (enumval == INT_MIN) { + sds enumerr = sdsnew("argument must be one of the following: "); + configEnum *enumNode = data.enumd.enum_value; + while(enumNode->name != NULL) { + enumerr = sdscatlen(enumerr, enumNode->name, + strlen(enumNode->name)); + enumerr = sdscatlen(enumerr, ", ", 2); + enumNode++; + } + sdsrange(enumerr,0,-3); /* Remove final ", ". */ + + strncpy(loadbuf, enumerr, LOADBUF_SIZE); + loadbuf[LOADBUF_SIZE - 1] = '\0'; + + sdsfree(enumerr); + *err = loadbuf; + return 0; + } + if (data.enumd.is_valid_fn && !data.enumd.is_valid_fn(enumval, err)) + return 0; + int prev = *(data.enumd.config); + *(data.enumd.config) = enumval; + if (update && data.enumd.update_fn && !data.enumd.update_fn(enumval, prev, err)) { + *(data.enumd.config) = prev; + return 0; + } + return 1; +} + +static void enumConfigGet(client *c, typeData data) { + addReplyBulkCString(c, configEnumGetNameOrUnknown(data.enumd.enum_value,*data.enumd.config)); +} + +static void enumConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + rewriteConfigEnumOption(state, name,*(data.enumd.config), data.enumd.enum_value, data.enumd.default_value); +} + +#define createEnumConfig(name, alias, modifiable, enum, config_addr, default, is_valid, update) { \ + embedCommonConfig(name, alias, modifiable) \ + embedConfigInterface(enumConfigInit, enumConfigSet, enumConfigGet, enumConfigRewrite) \ + .data.enumd = { \ + .config = &(config_addr), \ + .default_value = (default), \ + .is_valid_fn = (is_valid), \ + .update_fn = (update), \ + .enum_value = (enum), \ + } \ +} + +/* Gets a 'long long val' and sets it into the union, using a macro to get + * compile time type check. */ +#define SET_NUMERIC_TYPE(val) \ + if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { \ + *(data.numeric.config.i) = (int) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_UINT) { \ + *(data.numeric.config.ui) = (unsigned int) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG) { \ + *(data.numeric.config.l) = (long) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG) { \ + *(data.numeric.config.ul) = (unsigned long) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { \ + *(data.numeric.config.ll) = (long long) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG) { \ + *(data.numeric.config.ull) = (unsigned long long) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { \ + *(data.numeric.config.st) = (size_t) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SSIZE_T) { \ + *(data.numeric.config.sst) = (ssize_t) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_OFF_T) { \ + *(data.numeric.config.ot) = (off_t) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_TIME_T) { \ + *(data.numeric.config.tt) = (time_t) val; \ + } + +/* Gets a 'long long val' and sets it with the value from the union, using a + * macro to get compile time type check. */ +#define GET_NUMERIC_TYPE(val) \ + if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { \ + val = *(data.numeric.config.i); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_UINT) { \ + val = *(data.numeric.config.ui); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG) { \ + val = *(data.numeric.config.l); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG) { \ + val = *(data.numeric.config.ul); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { \ + val = *(data.numeric.config.ll); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG) { \ + val = *(data.numeric.config.ull); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { \ + val = *(data.numeric.config.st); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SSIZE_T) { \ + val = *(data.numeric.config.sst); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_OFF_T) { \ + val = *(data.numeric.config.ot); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_TIME_T) { \ + val = *(data.numeric.config.tt); \ + } + +/* Numeric configs */ +static void numericConfigInit(typeData data) { + SET_NUMERIC_TYPE(data.numeric.default_value) +} + +static int numericBoundaryCheck(typeData data, long long ll, char **err) { + if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG || + data.numeric.numeric_type == NUMERIC_TYPE_UINT || + data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { + /* Boundary check for unsigned types */ + unsigned long long ull = ll; + unsigned long long upper_bound = data.numeric.upper_bound; + unsigned long long lower_bound = data.numeric.lower_bound; + if (ull > upper_bound || ull < lower_bound) { + snprintf(loadbuf, LOADBUF_SIZE, + "argument must be between %llu and %llu inclusive", + lower_bound, + upper_bound); + *err = loadbuf; + return 0; + } + } else { + /* Boundary check for signed types */ + if (ll > data.numeric.upper_bound || ll < data.numeric.lower_bound) { + snprintf(loadbuf, LOADBUF_SIZE, + "argument must be between %lld and %lld inclusive", + data.numeric.lower_bound, + data.numeric.upper_bound); + *err = loadbuf; + return 0; + } + } + return 1; +} + +static int numericConfigSet(typeData data, sds value, int update, char **err) { + long long ll, prev = 0; + if (data.numeric.is_memory) { + int memerr; + ll = memtoll(value, &memerr); + if (memerr || ll < 0) { + *err = "argument must be a memory value"; + return 0; + } + } else { + if (!string2ll(value, sdslen(value),&ll)) { + *err = "argument couldn't be parsed into an integer" ; + return 0; + } + } + + if (!numericBoundaryCheck(data, ll, err)) + return 0; + + if (data.numeric.is_valid_fn && !data.numeric.is_valid_fn(ll, err)) + return 0; + + GET_NUMERIC_TYPE(prev) + SET_NUMERIC_TYPE(ll) + + if (update && data.numeric.update_fn && !data.numeric.update_fn(ll, prev, err)) { + SET_NUMERIC_TYPE(prev) + return 0; + } + return 1; +} + +static void numericConfigGet(client *c, typeData data) { + char buf[128]; + long long value = 0; + + GET_NUMERIC_TYPE(value) + + ll2string(buf, sizeof(buf), value); + addReplyBulkCString(c, buf); +} + +static void numericConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + long long value = 0; + + GET_NUMERIC_TYPE(value) + + if (data.numeric.is_memory) { + rewriteConfigBytesOption(state, name, value, data.numeric.default_value); + } else { + rewriteConfigNumericalOption(state, name, value, data.numeric.default_value); + } +} + +#define INTEGER_CONFIG 0 +#define MEMORY_CONFIG 1 + +#define embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) { \ + embedCommonConfig(name, alias, modifiable) \ + embedConfigInterface(numericConfigInit, numericConfigSet, numericConfigGet, numericConfigRewrite) \ + .data.numeric = { \ + .lower_bound = (lower), \ + .upper_bound = (upper), \ + .default_value = (default), \ + .is_valid_fn = (is_valid), \ + .update_fn = (update), \ + .is_memory = (memory), + +#define createIntConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_INT, \ + .config.i = &(config_addr) \ + } \ +} + +#define createUIntConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_UINT, \ + .config.ui = &(config_addr) \ + } \ +} + +#define createLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_LONG, \ + .config.l = &(config_addr) \ + } \ +} + +#define createULongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_ULONG, \ + .config.ul = &(config_addr) \ + } \ +} + +#define createLongLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_LONG_LONG, \ + .config.ll = &(config_addr) \ + } \ +} + +#define createULongLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_ULONG_LONG, \ + .config.ull = &(config_addr) \ + } \ +} + +#define createSizeTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_SIZE_T, \ + .config.st = &(config_addr) \ + } \ +} + +#define createSSizeTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_SSIZE_T, \ + .config.sst = &(config_addr) \ + } \ +} + +#define createTimeTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_TIME_T, \ + .config.tt = &(config_addr) \ + } \ +} + +#define createOffTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_OFF_T, \ + .config.ot = &(config_addr) \ + } \ +} + +static int isValidActiveDefrag(int val, char **err) { +#ifndef HAVE_DEFRAG + if (val) { + *err = "Active defragmentation cannot be enabled: it " + "requires a Redis server compiled with a modified Jemalloc " + "like the one shipped by default with the Redis source " + "distribution"; + return 0; + } +#else + UNUSED(val); + UNUSED(err); +#endif + return 1; +} + +static int isValidDBfilename(char *val, char **err) { + if (!pathIsBaseName(val)) { + *err = "dbfilename can't be a path, just a filename"; + return 0; + } + return 1; +} + +static int isValidAOFfilename(char *val, char **err) { + if (!pathIsBaseName(val)) { + *err = "appendfilename can't be a path, just a filename"; + return 0; + } + return 1; +} + +static int updateHZ(long long val, long long prev, char **err) { + UNUSED(prev); + UNUSED(err); + /* Hz is more an hint from the user, so we accept values out of range + * but cap them to reasonable values. */ + server.config_hz = val; + if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; + if (server.config_hz > CONFIG_MAX_HZ) server.config_hz = CONFIG_MAX_HZ; + server.hz = server.config_hz; + return 1; +} + +static int updateJemallocBgThread(int val, int prev, char **err) { + UNUSED(prev); + UNUSED(err); + set_jemalloc_bg_thread(val); + return 1; +} + +static int updateReplBacklogSize(long long val, long long prev, char **err) { + /* resizeReplicationBacklog sets server.repl_backlog_size, and relies on + * being able to tell when the size changes, so restore prev becore calling it. */ + UNUSED(err); + server.repl_backlog_size = prev; + resizeReplicationBacklog(val); + return 1; +} + +static int updateMaxmemory(long long val, long long prev, char **err) { + UNUSED(prev); + UNUSED(err); + if (val) { + size_t used = zmalloc_used_memory()-freeMemoryGetNotCountedMemory(); + if ((unsigned long long)val < used) { + serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET (%llu) is smaller than the current memory usage (%zu). This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy.", server.maxmemory, used); + } + freeMemoryIfNeededAndSafe(); + } + return 1; +} + +static int updateGoodSlaves(long long val, long long prev, char **err) { + UNUSED(val); + UNUSED(prev); + UNUSED(err); + refreshGoodSlavesCount(); + return 1; +} + +static int updateAppendonly(int val, int prev, char **err) { + UNUSED(prev); + if (val == 0 && server.aof_state != AOF_OFF) { + stopAppendOnly(); + } else if (val && server.aof_state == AOF_OFF) { + if (startAppendOnly() == C_ERR) { + *err = "Unable to turn on AOF. Check server logs."; + return 0; + } + } + return 1; +} + +static int updateMaxclients(long long val, long long prev, char **err) { + /* Try to check if the OS is capable of supporting so many FDs. */ + if (val > prev) { + adjustOpenFilesLimit(); + if (server.maxclients != val) { + static char msg[128]; + sprintf(msg, "The operating system is not able to handle the specified number of clients, try with %d", server.maxclients); + *err = msg; + if (server.maxclients > prev) { + server.maxclients = prev; + adjustOpenFilesLimit(); + } + return 0; + } + if ((unsigned int) aeGetSetSize(server.el) < + server.maxclients + CONFIG_FDSET_INCR) + { + if (aeResizeSetSize(server.el, + server.maxclients + CONFIG_FDSET_INCR) == AE_ERR) + { + *err = "The event loop API used by Redis is not able to handle the specified number of clients"; + return 0; + } + } + } + return 1; +} + +#ifdef USE_OPENSSL +static int updateTlsCfg(char *val, char *prev, char **err) { + UNUSED(val); + UNUSED(prev); + UNUSED(err); + if (tlsConfigure(&server.tls_ctx_config) == C_ERR) { + *err = "Unable to configure tls-cert-file. Check server logs."; + return 0; + } + return 1; +} +static int updateTlsCfgBool(int val, int prev, char **err) { + UNUSED(val); + UNUSED(prev); + return updateTlsCfg(NULL, NULL, err); +} +#endif /* USE_OPENSSL */ + +standardConfig configs[] = { + /* Bool configs */ + createBoolConfig("rdbchecksum", NULL, IMMUTABLE_CONFIG, server.rdb_checksum, 1, NULL, NULL), + createBoolConfig("daemonize", NULL, IMMUTABLE_CONFIG, server.daemonize, 0, NULL, NULL), + createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, server.io_threads_do_reads, 0,NULL, NULL), /* Read + parse from threads? */ + createBoolConfig("lua-replicate-commands", NULL, MODIFIABLE_CONFIG, server.lua_always_replicate_commands, 1, NULL, NULL), + createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, 0, NULL, NULL), + createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, 1, NULL, NULL), + createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, 1, NULL, NULL), + createBoolConfig("rdb-del-sync-files", NULL, MODIFIABLE_CONFIG, server.rdb_del_sync_files, 0, NULL, NULL), + createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, 1, NULL, NULL), + createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, 1, NULL, NULL), + createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, server.dynamic_hz, 1, NULL, NULL), /* Adapt hz to # of clients.*/ + createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, 0, NULL, NULL), + createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, 0, NULL, NULL), + createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_server_del, 0, NULL, NULL), + createBoolConfig("lazyfree-lazy-user-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_user_del , 0, NULL, NULL), + createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, server.repl_disable_tcp_nodelay, 0, NULL, NULL), + createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, server.repl_diskless_sync, 0, NULL, NULL), + createBoolConfig("gopher-enabled", NULL, MODIFIABLE_CONFIG, server.gopher_enabled, 0, NULL, NULL), + createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.aof_rewrite_incremental_fsync, 1, NULL, NULL), + createBoolConfig("no-appendfsync-on-rewrite", NULL, MODIFIABLE_CONFIG, server.aof_no_fsync_on_rewrite, 0, NULL, NULL), + createBoolConfig("cluster-require-full-coverage", NULL, MODIFIABLE_CONFIG, server.cluster_require_full_coverage, 1, NULL, NULL), + createBoolConfig("rdb-save-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.rdb_save_incremental_fsync, 1, NULL, NULL), + createBoolConfig("aof-load-truncated", NULL, MODIFIABLE_CONFIG, server.aof_load_truncated, 1, NULL, NULL), + createBoolConfig("aof-use-rdb-preamble", NULL, MODIFIABLE_CONFIG, server.aof_use_rdb_preamble, 1, NULL, NULL), + createBoolConfig("cluster-replica-no-failover", "cluster-slave-no-failover", MODIFIABLE_CONFIG, server.cluster_slave_no_failover, 0, NULL, NULL), /* Failover by default. */ + createBoolConfig("replica-lazy-flush", "slave-lazy-flush", MODIFIABLE_CONFIG, server.repl_slave_lazy_flush, 0, NULL, NULL), + createBoolConfig("replica-serve-stale-data", "slave-serve-stale-data", MODIFIABLE_CONFIG, server.repl_serve_stale_data, 1, NULL, NULL), + createBoolConfig("replica-read-only", "slave-read-only", MODIFIABLE_CONFIG, server.repl_slave_ro, 1, NULL, NULL), + createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, server.repl_slave_ignore_maxmemory, 1, NULL, NULL), + createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, server.jemalloc_bg_thread, 1, NULL, updateJemallocBgThread), + createBoolConfig("activedefrag", NULL, MODIFIABLE_CONFIG, server.active_defrag_enabled, 0, isValidActiveDefrag, NULL), + createBoolConfig("syslog-enabled", NULL, IMMUTABLE_CONFIG, server.syslog_enabled, 0, NULL, NULL), + createBoolConfig("cluster-enabled", NULL, IMMUTABLE_CONFIG, server.cluster_enabled, 0, NULL, NULL), + createBoolConfig("appendonly", NULL, MODIFIABLE_CONFIG, server.aof_enabled, 0, NULL, updateAppendonly), + createBoolConfig("cluster-allow-reads-when-down", NULL, MODIFIABLE_CONFIG, server.cluster_allow_reads_when_down, 0, NULL, NULL), + + + /* String Configs */ + createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, "", NULL, NULL), + createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.unixsocket, NULL, NULL, NULL), + createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.pidfile, NULL, NULL, NULL), + createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.slave_announce_ip, NULL, NULL, NULL), + createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masteruser, NULL, NULL, NULL), + createStringConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL, NULL, NULL), + createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL, NULL, NULL), + createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.syslog_ident, "redis", NULL, NULL), + createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.rdb_filename, "dump.rdb", isValidDBfilename, NULL), + createStringConfig("appendfilename", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.aof_filename, "appendonly.aof", isValidAOFfilename, NULL), + createStringConfig("server_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.server_cpulist, NULL, NULL, NULL), + createStringConfig("bio_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bio_cpulist, NULL, NULL, NULL), + createStringConfig("aof_rewrite_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.aof_rewrite_cpulist, NULL, NULL, NULL), + createStringConfig("bgsave_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bgsave_cpulist, NULL, NULL, NULL), + + /* Enum Configs */ + createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE, NULL, NULL), + createEnumConfig("syslog-facility", NULL, IMMUTABLE_CONFIG, syslog_facility_enum, server.syslog_facility, LOG_LOCAL0, NULL, NULL), + createEnumConfig("repl-diskless-load", NULL, MODIFIABLE_CONFIG, repl_diskless_load_enum, server.repl_diskless_load, REPL_DISKLESS_LOAD_DISABLED, NULL, NULL), + createEnumConfig("loglevel", NULL, MODIFIABLE_CONFIG, loglevel_enum, server.verbosity, LL_NOTICE, NULL, NULL), + createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, server.maxmemory_policy, MAXMEMORY_NO_EVICTION, NULL, NULL), + createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, AOF_FSYNC_EVERYSEC, NULL, NULL), + + /* Integer configs */ + createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, 16, INTEGER_CONFIG, NULL, NULL), + createIntConfig("port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.port, 6379, INTEGER_CONFIG, NULL, NULL), /* TCP port. */ + createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 128, server.io_threads_num, 1, INTEGER_CONFIG, NULL, NULL), /* Single threaded by default */ + createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.aof_rewrite_perc, 100, INTEGER_CONFIG, NULL, NULL), + createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_slave_validity_factor, 10, INTEGER_CONFIG, NULL, NULL), /* Slave max data age factor. */ + createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.list_max_ziplist_size, -2, INTEGER_CONFIG, NULL, NULL), + createIntConfig("tcp-keepalive", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcpkeepalive, 300, INTEGER_CONFIG, NULL, NULL), + createIntConfig("cluster-migration-barrier", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_migration_barrier, 1, INTEGER_CONFIG, NULL, NULL), + createIntConfig("active-defrag-cycle-min", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_min, 1, INTEGER_CONFIG, NULL, NULL), /* Default: 1% CPU min (at lower threshold) */ + createIntConfig("active-defrag-cycle-max", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_max, 25, INTEGER_CONFIG, NULL, NULL), /* Default: 25% CPU max (at upper threshold) */ + createIntConfig("active-defrag-threshold-lower", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_lower, 10, INTEGER_CONFIG, NULL, NULL), /* Default: don't defrag when fragmentation is below 10% */ + createIntConfig("active-defrag-threshold-upper", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_upper, 100, INTEGER_CONFIG, NULL, NULL), /* Default: maximum defrag force at 100% fragmentation */ + createIntConfig("lfu-log-factor", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_log_factor, 10, INTEGER_CONFIG, NULL, NULL), + createIntConfig("lfu-decay-time", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_decay_time, 1, INTEGER_CONFIG, NULL, NULL), + createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, server.slave_priority, 100, INTEGER_CONFIG, NULL, NULL), + createIntConfig("repl-diskless-sync-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_diskless_sync_delay, 5, INTEGER_CONFIG, NULL, NULL), + createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.maxmemory_samples, 5, INTEGER_CONFIG, NULL, NULL), + createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.maxidletime, 0, INTEGER_CONFIG, NULL, NULL), /* Default client timeout: infinite */ + createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, server.slave_announce_port, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("tcp-backlog", NULL, IMMUTABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, 511, INTEGER_CONFIG, NULL, NULL), /* TCP listen backlog. */ + createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_bus_port, 0, INTEGER_CONFIG, NULL, NULL), /* Default: Use +10000 offset. */ + createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_port, 0, INTEGER_CONFIG, NULL, NULL), /* Use server.port */ + createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_timeout, 60, INTEGER_CONFIG, NULL, NULL), + createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_ping_slave_period, 10, INTEGER_CONFIG, NULL, NULL), + createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, 1, INTEGER_CONFIG, NULL, NULL), /* From 1 to 10. */ + createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG, NULL, updateHZ), + createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves), + createIntConfig("min-replicas-max-lag", "min-slaves-max-lag", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_max_lag, 10, INTEGER_CONFIG, NULL, updateGoodSlaves), + + /* Unsigned int configs */ + createUIntConfig("maxclients", NULL, MODIFIABLE_CONFIG, 1, UINT_MAX, server.maxclients, 10000, INTEGER_CONFIG, NULL, updateMaxclients), + + /* Unsigned Long configs */ + createULongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, 1000, INTEGER_CONFIG, NULL, NULL), /* Default: keys with more than 1000 fields will be processed separately */ + createULongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, 128, INTEGER_CONFIG, NULL, NULL), + createULongConfig("acllog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.acllog_max_len, 128, INTEGER_CONFIG, NULL, NULL), + + /* Long Long configs */ + createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, 5000, INTEGER_CONFIG, NULL, NULL),/* milliseconds */ + createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, 15000, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, 10000, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, 0, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */ + createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.stream_node_max_entries, 100, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */ + + /* Unsigned Long Long configs */ + createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.maxmemory, 0, MEMORY_CONFIG, NULL, updateMaxmemory), + + /* Size_t configs */ + createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_entries, 512, INTEGER_CONFIG, NULL, NULL), + createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, 512, INTEGER_CONFIG, NULL, NULL), + createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_entries, 128, INTEGER_CONFIG, NULL, NULL), + createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.active_defrag_ignore_bytes, 100<<20, MEMORY_CONFIG, NULL, NULL), /* Default: don't defrag if frag overhead is below 100mb */ + createSizeTConfig("hash-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, 4096, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, 3000, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("tracking-table-max-keys", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.tracking_table_max_keys, 1000000, INTEGER_CONFIG, NULL, NULL), /* Default: 1 million keys max. */ + + /* Other configs */ + createTimeTConfig("repl-backlog-ttl", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.repl_backlog_time_limit, 60*60, INTEGER_CONFIG, NULL, NULL), /* Default: 1 hour */ + createOffTConfig("auto-aof-rewrite-min-size", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.aof_rewrite_min_size, 64*1024*1024, MEMORY_CONFIG, NULL, NULL), + +#ifdef USE_OPENSSL + createIntConfig("tls-port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.tls_port, 0, INTEGER_CONFIG, NULL, NULL), /* TCP port. */ + createBoolConfig("tls-cluster", NULL, MODIFIABLE_CONFIG, server.tls_cluster, 0, NULL, NULL), + createBoolConfig("tls-replication", NULL, MODIFIABLE_CONFIG, server.tls_replication, 0, NULL, NULL), + createBoolConfig("tls-auth-clients", NULL, MODIFIABLE_CONFIG, server.tls_auth_clients, 1, NULL, NULL), + createBoolConfig("tls-prefer-server-ciphers", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.prefer_server_ciphers, 0, NULL, updateTlsCfgBool), + createStringConfig("tls-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.cert_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.key_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-dh-params-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.dh_params_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ca-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ca_cert_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ca-cert-dir", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ca_cert_dir, NULL, NULL, updateTlsCfg), + createStringConfig("tls-protocols", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.protocols, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ciphers", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ciphers, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ciphersuites", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ciphersuites, NULL, NULL, updateTlsCfg), +#endif + + /* NULL Terminator */ + {NULL} +}; + /*----------------------------------------------------------------------------- * CONFIG command entry point *----------------------------------------------------------------------------*/ diff --git a/redis.submodule/src/config.h b/redis.submodule/src/config.h index efa9d11..0fcc429 100644 --- a/redis.submodule/src/config.h +++ b/redis.submodule/src/config.h @@ -226,4 +226,31 @@ void setproctitle(const char *fmt, ...); #define USE_ALIGNED_ACCESS #endif +/* Define for redis_set_thread_title */ +#ifdef __linux__ +#define redis_set_thread_title(name) pthread_setname_np(pthread_self(), name) +#else +#if (defined __FreeBSD__ || defined __OpenBSD__) +#include +#define redis_set_thread_title(name) pthread_set_name_np(pthread_self(), name) +#elif defined __NetBSD__ +#include +#define redis_set_thread_title(name) pthread_setname_np(pthread_self(), name, NULL) +#else +#if (defined __APPLE__ && defined(MAC_OS_X_VERSION_10_7)) +int pthread_setname_np(const char *name); +#include +#define redis_set_thread_title(name) pthread_setname_np(name) +#else +#define redis_set_thread_title(name) +#endif +#endif +#endif + +/* Check if we can use setcpuaffinity(). */ +#if (defined __linux || defined __NetBSD__ || defined __FreeBSD__) +#define USE_SETCPUAFFINITY +void setcpuaffinity(const char *cpulist); +#endif + #endif diff --git a/redis.submodule/src/connection.c b/redis.submodule/src/connection.c new file mode 100644 index 0000000..2015c91 --- /dev/null +++ b/redis.submodule/src/connection.c @@ -0,0 +1,413 @@ +/* + * Copyright (c) 2019, Redis Labs + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "server.h" +#include "connhelpers.h" + +/* The connections module provides a lean abstraction of network connections + * to avoid direct socket and async event management across the Redis code base. + * + * It does NOT provide advanced connection features commonly found in similar + * libraries such as complete in/out buffer management, throttling, etc. These + * functions remain in networking.c. + * + * The primary goal is to allow transparent handling of TCP and TLS based + * connections. To do so, connections have the following properties: + * + * 1. A connection may live before its corresponding socket exists. This + * allows various context and configuration setting to be handled before + * establishing the actual connection. + * 2. The caller may register/unregister logical read/write handlers to be + * called when the connection has data to read from/can accept writes. + * These logical handlers may or may not correspond to actual AE events, + * depending on the implementation (for TCP they are; for TLS they aren't). + */ + +ConnectionType CT_Socket; + +/* When a connection is created we must know its type already, but the + * underlying socket may or may not exist: + * + * - For accepted connections, it exists as we do not model the listen/accept + * part; So caller calls connCreateSocket() followed by connAccept(). + * - For outgoing connections, the socket is created by the connection module + * itself; So caller calls connCreateSocket() followed by connConnect(), + * which registers a connect callback that fires on connected/error state + * (and after any transport level handshake was done). + * + * NOTE: An earlier version relied on connections being part of other structs + * and not independently allocated. This could lead to further optimizations + * like using container_of(), etc. However it was discontinued in favor of + * this approach for these reasons: + * + * 1. In some cases conns are created/handled outside the context of the + * containing struct, in which case it gets a bit awkward to copy them. + * 2. Future implementations may wish to allocate arbitrary data for the + * connection. + * 3. The container_of() approach is anyway risky because connections may + * be embedded in different structs, not just client. + */ + +connection *connCreateSocket() { + connection *conn = zcalloc(sizeof(connection)); + conn->type = &CT_Socket; + conn->fd = -1; + + return conn; +} + +/* Create a new socket-type connection that is already associated with + * an accepted connection. + * + * The socket is not read for I/O until connAccept() was called and + * invoked the connection-level accept handler. + */ +connection *connCreateAcceptedSocket(int fd) { + connection *conn = connCreateSocket(); + conn->fd = fd; + conn->state = CONN_STATE_ACCEPTING; + return conn; +} + +static int connSocketConnect(connection *conn, const char *addr, int port, const char *src_addr, + ConnectionCallbackFunc connect_handler) { + int fd = anetTcpNonBlockBestEffortBindConnect(NULL,addr,port,src_addr); + if (fd == -1) { + conn->state = CONN_STATE_ERROR; + conn->last_errno = errno; + return C_ERR; + } + + conn->fd = fd; + conn->state = CONN_STATE_CONNECTING; + + conn->conn_handler = connect_handler; + aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE, + conn->type->ae_handler, conn); + + return C_OK; +} + +/* Returns true if a write handler is registered */ +int connHasWriteHandler(connection *conn) { + return conn->write_handler != NULL; +} + +/* Returns true if a read handler is registered */ +int connHasReadHandler(connection *conn) { + return conn->read_handler != NULL; +} + +/* Associate a private data pointer with the connection */ +void connSetPrivateData(connection *conn, void *data) { + conn->private_data = data; +} + +/* Get the associated private data pointer */ +void *connGetPrivateData(connection *conn) { + return conn->private_data; +} + +/* ------ Pure socket connections ------- */ + +/* A very incomplete list of implementation-specific calls. Much of the above shall + * move here as we implement additional connection types. + */ + +/* Close the connection and free resources. */ +static void connSocketClose(connection *conn) { + if (conn->fd != -1) { + aeDeleteFileEvent(server.el,conn->fd,AE_READABLE); + aeDeleteFileEvent(server.el,conn->fd,AE_WRITABLE); + close(conn->fd); + conn->fd = -1; + } + + /* If called from within a handler, schedule the close but + * keep the connection until the handler returns. + */ + if (connHasRefs(conn)) { + conn->flags |= CONN_FLAG_CLOSE_SCHEDULED; + return; + } + + zfree(conn); +} + +static int connSocketWrite(connection *conn, const void *data, size_t data_len) { + int ret = write(conn->fd, data, data_len); + if (ret < 0 && errno != EAGAIN) { + conn->last_errno = errno; + conn->state = CONN_STATE_ERROR; + } + + return ret; +} + +static int connSocketRead(connection *conn, void *buf, size_t buf_len) { + int ret = read(conn->fd, buf, buf_len); + if (!ret) { + conn->state = CONN_STATE_CLOSED; + } else if (ret < 0 && errno != EAGAIN) { + conn->last_errno = errno; + conn->state = CONN_STATE_ERROR; + } + + return ret; +} + +static int connSocketAccept(connection *conn, ConnectionCallbackFunc accept_handler) { + int ret = C_OK; + + if (conn->state != CONN_STATE_ACCEPTING) return C_ERR; + conn->state = CONN_STATE_CONNECTED; + + connIncrRefs(conn); + if (!callHandler(conn, accept_handler)) ret = C_ERR; + connDecrRefs(conn); + + return ret; +} + +/* Register a write handler, to be called when the connection is writable. + * If NULL, the existing handler is removed. + * + * The barrier flag indicates a write barrier is requested, resulting with + * CONN_FLAG_WRITE_BARRIER set. This will ensure that the write handler is + * always called before and not after the read handler in a single event + * loop. + */ +static int connSocketSetWriteHandler(connection *conn, ConnectionCallbackFunc func, int barrier) { + if (func == conn->write_handler) return C_OK; + + conn->write_handler = func; + if (barrier) + conn->flags |= CONN_FLAG_WRITE_BARRIER; + else + conn->flags &= ~CONN_FLAG_WRITE_BARRIER; + if (!conn->write_handler) + aeDeleteFileEvent(server.el,conn->fd,AE_WRITABLE); + else + if (aeCreateFileEvent(server.el,conn->fd,AE_WRITABLE, + conn->type->ae_handler,conn) == AE_ERR) return C_ERR; + return C_OK; +} + +/* Register a read handler, to be called when the connection is readable. + * If NULL, the existing handler is removed. + */ +static int connSocketSetReadHandler(connection *conn, ConnectionCallbackFunc func) { + if (func == conn->read_handler) return C_OK; + + conn->read_handler = func; + if (!conn->read_handler) + aeDeleteFileEvent(server.el,conn->fd,AE_READABLE); + else + if (aeCreateFileEvent(server.el,conn->fd, + AE_READABLE,conn->type->ae_handler,conn) == AE_ERR) return C_ERR; + return C_OK; +} + +static const char *connSocketGetLastError(connection *conn) { + return strerror(conn->last_errno); +} + +static void connSocketEventHandler(struct aeEventLoop *el, int fd, void *clientData, int mask) +{ + UNUSED(el); + UNUSED(fd); + connection *conn = clientData; + + if (conn->state == CONN_STATE_CONNECTING && + (mask & AE_WRITABLE) && conn->conn_handler) { + + if (connGetSocketError(conn)) { + conn->last_errno = errno; + conn->state = CONN_STATE_ERROR; + } else { + conn->state = CONN_STATE_CONNECTED; + } + + if (!conn->write_handler) aeDeleteFileEvent(server.el,conn->fd,AE_WRITABLE); + + if (!callHandler(conn, conn->conn_handler)) return; + conn->conn_handler = NULL; + } + + /* Normally we execute the readable event first, and the writable + * event later. This is useful as sometimes we may be able + * to serve the reply of a query immediately after processing the + * query. + * + * However if WRITE_BARRIER is set in the mask, our application is + * asking us to do the reverse: never fire the writable event + * after the readable. In such a case, we invert the calls. + * This is useful when, for instance, we want to do things + * in the beforeSleep() hook, like fsync'ing a file to disk, + * before replying to a client. */ + int invert = conn->flags & CONN_FLAG_WRITE_BARRIER; + + int call_write = (mask & AE_WRITABLE) && conn->write_handler; + int call_read = (mask & AE_READABLE) && conn->read_handler; + + /* Handle normal I/O flows */ + if (!invert && call_read) { + if (!callHandler(conn, conn->read_handler)) return; + } + /* Fire the writable event. */ + if (call_write) { + if (!callHandler(conn, conn->write_handler)) return; + } + /* If we have to invert the call, fire the readable event now + * after the writable one. */ + if (invert && call_read) { + if (!callHandler(conn, conn->read_handler)) return; + } +} + +static int connSocketBlockingConnect(connection *conn, const char *addr, int port, long long timeout) { + int fd = anetTcpNonBlockConnect(NULL,addr,port); + if (fd == -1) { + conn->state = CONN_STATE_ERROR; + conn->last_errno = errno; + return C_ERR; + } + + if ((aeWait(fd, AE_WRITABLE, timeout) & AE_WRITABLE) == 0) { + conn->state = CONN_STATE_ERROR; + conn->last_errno = ETIMEDOUT; + } + + conn->fd = fd; + conn->state = CONN_STATE_CONNECTED; + return C_OK; +} + +/* Connection-based versions of syncio.c functions. + * NOTE: This should ideally be refactored out in favor of pure async work. + */ + +static ssize_t connSocketSyncWrite(connection *conn, char *ptr, ssize_t size, long long timeout) { + return syncWrite(conn->fd, ptr, size, timeout); +} + +static ssize_t connSocketSyncRead(connection *conn, char *ptr, ssize_t size, long long timeout) { + return syncRead(conn->fd, ptr, size, timeout); +} + +static ssize_t connSocketSyncReadLine(connection *conn, char *ptr, ssize_t size, long long timeout) { + return syncReadLine(conn->fd, ptr, size, timeout); +} + + +ConnectionType CT_Socket = { + .ae_handler = connSocketEventHandler, + .close = connSocketClose, + .write = connSocketWrite, + .read = connSocketRead, + .accept = connSocketAccept, + .connect = connSocketConnect, + .set_write_handler = connSocketSetWriteHandler, + .set_read_handler = connSocketSetReadHandler, + .get_last_error = connSocketGetLastError, + .blocking_connect = connSocketBlockingConnect, + .sync_write = connSocketSyncWrite, + .sync_read = connSocketSyncRead, + .sync_readline = connSocketSyncReadLine +}; + + +int connGetSocketError(connection *conn) { + int sockerr = 0; + socklen_t errlen = sizeof(sockerr); + + if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, &sockerr, &errlen) == -1) + sockerr = errno; + return sockerr; +} + +int connPeerToString(connection *conn, char *ip, size_t ip_len, int *port) { + return anetPeerToString(conn ? conn->fd : -1, ip, ip_len, port); +} + +int connFormatPeer(connection *conn, char *buf, size_t buf_len) { + return anetFormatPeer(conn ? conn->fd : -1, buf, buf_len); +} + +int connSockName(connection *conn, char *ip, size_t ip_len, int *port) { + return anetSockName(conn->fd, ip, ip_len, port); +} + +int connBlock(connection *conn) { + if (conn->fd == -1) return C_ERR; + return anetBlock(NULL, conn->fd); +} + +int connNonBlock(connection *conn) { + if (conn->fd == -1) return C_ERR; + return anetNonBlock(NULL, conn->fd); +} + +int connEnableTcpNoDelay(connection *conn) { + if (conn->fd == -1) return C_ERR; + return anetEnableTcpNoDelay(NULL, conn->fd); +} + +int connDisableTcpNoDelay(connection *conn) { + if (conn->fd == -1) return C_ERR; + return anetDisableTcpNoDelay(NULL, conn->fd); +} + +int connKeepAlive(connection *conn, int interval) { + if (conn->fd == -1) return C_ERR; + return anetKeepAlive(NULL, conn->fd, interval); +} + +int connSendTimeout(connection *conn, long long ms) { + return anetSendTimeout(NULL, conn->fd, ms); +} + +int connRecvTimeout(connection *conn, long long ms) { + return anetRecvTimeout(NULL, conn->fd, ms); +} + +int connGetState(connection *conn) { + return conn->state; +} + +/* Return a text that describes the connection, suitable for inclusion + * in CLIENT LIST and similar outputs. + * + * For sockets, we always return "fd=" to maintain compatibility. + */ +const char *connGetInfo(connection *conn, char *buf, size_t buf_len) { + snprintf(buf, buf_len-1, "fd=%i", conn->fd); + return buf; +} + diff --git a/redis.submodule/src/connection.h b/redis.submodule/src/connection.h new file mode 100644 index 0000000..0fd6c5f --- /dev/null +++ b/redis.submodule/src/connection.h @@ -0,0 +1,227 @@ + +/* + * Copyright (c) 2019, Redis Labs + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __REDIS_CONNECTION_H +#define __REDIS_CONNECTION_H + +#define CONN_INFO_LEN 32 + +struct aeEventLoop; +typedef struct connection connection; + +typedef enum { + CONN_STATE_NONE = 0, + CONN_STATE_CONNECTING, + CONN_STATE_ACCEPTING, + CONN_STATE_CONNECTED, + CONN_STATE_CLOSED, + CONN_STATE_ERROR +} ConnectionState; + +#define CONN_FLAG_CLOSE_SCHEDULED (1<<0) /* Closed scheduled by a handler */ +#define CONN_FLAG_WRITE_BARRIER (1<<1) /* Write barrier requested */ + +typedef void (*ConnectionCallbackFunc)(struct connection *conn); + +typedef struct ConnectionType { + void (*ae_handler)(struct aeEventLoop *el, int fd, void *clientData, int mask); + int (*connect)(struct connection *conn, const char *addr, int port, const char *source_addr, ConnectionCallbackFunc connect_handler); + int (*write)(struct connection *conn, const void *data, size_t data_len); + int (*read)(struct connection *conn, void *buf, size_t buf_len); + void (*close)(struct connection *conn); + int (*accept)(struct connection *conn, ConnectionCallbackFunc accept_handler); + int (*set_write_handler)(struct connection *conn, ConnectionCallbackFunc handler, int barrier); + int (*set_read_handler)(struct connection *conn, ConnectionCallbackFunc handler); + const char *(*get_last_error)(struct connection *conn); + int (*blocking_connect)(struct connection *conn, const char *addr, int port, long long timeout); + ssize_t (*sync_write)(struct connection *conn, char *ptr, ssize_t size, long long timeout); + ssize_t (*sync_read)(struct connection *conn, char *ptr, ssize_t size, long long timeout); + ssize_t (*sync_readline)(struct connection *conn, char *ptr, ssize_t size, long long timeout); +} ConnectionType; + +struct connection { + ConnectionType *type; + ConnectionState state; + short int flags; + short int refs; + int last_errno; + void *private_data; + ConnectionCallbackFunc conn_handler; + ConnectionCallbackFunc write_handler; + ConnectionCallbackFunc read_handler; + int fd; +}; + +/* The connection module does not deal with listening and accepting sockets, + * so we assume we have a socket when an incoming connection is created. + * + * The fd supplied should therefore be associated with an already accept()ed + * socket. + * + * connAccept() may directly call accept_handler(), or return and call it + * at a later time. This behavior is a bit awkward but aims to reduce the need + * to wait for the next event loop, if no additional handshake is required. + * + * IMPORTANT: accept_handler may decide to close the connection, calling connClose(). + * To make this safe, the connection is only marked with CONN_FLAG_CLOSE_SCHEDULED + * in this case, and connAccept() returns with an error. + * + * connAccept() callers must always check the return value and on error (C_ERR) + * a connClose() must be called. + */ + +static inline int connAccept(connection *conn, ConnectionCallbackFunc accept_handler) { + return conn->type->accept(conn, accept_handler); +} + +/* Establish a connection. The connect_handler will be called when the connection + * is established, or if an error has occured. + * + * The connection handler will be responsible to set up any read/write handlers + * as needed. + * + * If C_ERR is returned, the operation failed and the connection handler shall + * not be expected. + */ +static inline int connConnect(connection *conn, const char *addr, int port, const char *src_addr, + ConnectionCallbackFunc connect_handler) { + return conn->type->connect(conn, addr, port, src_addr, connect_handler); +} + +/* Blocking connect. + * + * NOTE: This is implemented in order to simplify the transition to the abstract + * connections, but should probably be refactored out of cluster.c and replication.c, + * in favor of a pure async implementation. + */ +static inline int connBlockingConnect(connection *conn, const char *addr, int port, long long timeout) { + return conn->type->blocking_connect(conn, addr, port, timeout); +} + +/* Write to connection, behaves the same as write(2). + * + * Like write(2), a short write is possible. A -1 return indicates an error. + * + * The caller should NOT rely on errno. Testing for an EAGAIN-like condition, use + * connGetState() to see if the connection state is still CONN_STATE_CONNECTED. + */ +static inline int connWrite(connection *conn, const void *data, size_t data_len) { + return conn->type->write(conn, data, data_len); +} + +/* Read from the connection, behaves the same as read(2). + * + * Like read(2), a short read is possible. A return value of 0 will indicate the + * connection was closed, and -1 will indicate an error. + * + * The caller should NOT rely on errno. Testing for an EAGAIN-like condition, use + * connGetState() to see if the connection state is still CONN_STATE_CONNECTED. + */ +static inline int connRead(connection *conn, void *buf, size_t buf_len) { + return conn->type->read(conn, buf, buf_len); +} + +/* Register a write handler, to be called when the connection is writable. + * If NULL, the existing handler is removed. + */ +static inline int connSetWriteHandler(connection *conn, ConnectionCallbackFunc func) { + return conn->type->set_write_handler(conn, func, 0); +} + +/* Register a read handler, to be called when the connection is readable. + * If NULL, the existing handler is removed. + */ +static inline int connSetReadHandler(connection *conn, ConnectionCallbackFunc func) { + return conn->type->set_read_handler(conn, func); +} + +/* Set a write handler, and possibly enable a write barrier, this flag is + * cleared when write handler is changed or removed. + * With barroer enabled, we never fire the event if the read handler already + * fired in the same event loop iteration. Useful when you want to persist + * things to disk before sending replies, and want to do that in a group fashion. */ +static inline int connSetWriteHandlerWithBarrier(connection *conn, ConnectionCallbackFunc func, int barrier) { + return conn->type->set_write_handler(conn, func, barrier); +} + +static inline void connClose(connection *conn) { + conn->type->close(conn); +} + +/* Returns the last error encountered by the connection, as a string. If no error, + * a NULL is returned. + */ +static inline const char *connGetLastError(connection *conn) { + return conn->type->get_last_error(conn); +} + +static inline ssize_t connSyncWrite(connection *conn, char *ptr, ssize_t size, long long timeout) { + return conn->type->sync_write(conn, ptr, size, timeout); +} + +static inline ssize_t connSyncRead(connection *conn, char *ptr, ssize_t size, long long timeout) { + return conn->type->sync_read(conn, ptr, size, timeout); +} + +static inline ssize_t connSyncReadLine(connection *conn, char *ptr, ssize_t size, long long timeout) { + return conn->type->sync_readline(conn, ptr, size, timeout); +} + +connection *connCreateSocket(); +connection *connCreateAcceptedSocket(int fd); + +connection *connCreateTLS(); +connection *connCreateAcceptedTLS(int fd, int require_auth); + +void connSetPrivateData(connection *conn, void *data); +void *connGetPrivateData(connection *conn); +int connGetState(connection *conn); +int connHasWriteHandler(connection *conn); +int connHasReadHandler(connection *conn); +int connGetSocketError(connection *conn); + +/* anet-style wrappers to conns */ +int connBlock(connection *conn); +int connNonBlock(connection *conn); +int connEnableTcpNoDelay(connection *conn); +int connDisableTcpNoDelay(connection *conn); +int connKeepAlive(connection *conn, int interval); +int connSendTimeout(connection *conn, long long ms); +int connRecvTimeout(connection *conn, long long ms); +int connPeerToString(connection *conn, char *ip, size_t ip_len, int *port); +int connFormatPeer(connection *conn, char *buf, size_t buf_len); +int connSockName(connection *conn, char *ip, size_t ip_len, int *port); +const char *connGetInfo(connection *conn, char *buf, size_t buf_len); + +/* Helpers for tls special considerations */ +int tlsHasPendingData(); +int tlsProcessPendingData(); + +#endif /* __REDIS_CONNECTION_H */ diff --git a/redis.submodule/src/connhelpers.h b/redis.submodule/src/connhelpers.h new file mode 100644 index 0000000..86250d0 --- /dev/null +++ b/redis.submodule/src/connhelpers.h @@ -0,0 +1,88 @@ + +/* + * Copyright (c) 2019, Redis Labs + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __REDIS_CONNHELPERS_H +#define __REDIS_CONNHELPERS_H + +#include "connection.h" + +/* These are helper functions that are common to different connection + * implementations (currently sockets in connection.c and TLS in tls.c). + * + * Currently helpers implement the mechanisms for invoking connection + * handlers and tracking connection references, to allow safe destruction + * of connections from within a handler. + */ + +/* Incremenet connection references. + * + * Inside a connection handler, we guarantee refs >= 1 so it is always + * safe to connClose(). + * + * In other cases where we don't want to prematurely lose the connection, + * it can go beyond 1 as well; currently it is only done by connAccept(). + */ +static inline void connIncrRefs(connection *conn) { + conn->refs++; +} + +/* Decrement connection references. + * + * Note that this is not intended to provide any automatic free logic! + * callHandler() takes care of that for the common flows, and anywhere an + * explicit connIncrRefs() is used, the caller is expected to take care of + * that. + */ + +static inline void connDecrRefs(connection *conn) { + conn->refs--; +} + +static inline int connHasRefs(connection *conn) { + return conn->refs; +} + +/* Helper for connection implementations to call handlers: + * 1. Increment refs to protect the connection. + * 2. Execute the handler (if set). + * 3. Decrement refs and perform deferred close, if refs==0. + */ +static inline int callHandler(connection *conn, ConnectionCallbackFunc handler) { + connIncrRefs(conn); + if (handler) handler(conn); + connDecrRefs(conn); + if (conn->flags & CONN_FLAG_CLOSE_SCHEDULED) { + if (!connHasRefs(conn)) connClose(conn); + return 0; + } + return 1; +} + +#endif /* __REDIS_CONNHELPERS_H */ diff --git a/redis.submodule/src/crc16_slottable.h b/redis.submodule/src/crc16_slottable.h new file mode 100644 index 0000000..652aea9 --- /dev/null +++ b/redis.submodule/src/crc16_slottable.h @@ -0,0 +1,835 @@ +#ifndef _CRC16_TABLE_H__ +#define _CRC16_TABLE_H__ + +/* A table of the shortest possible alphanumeric string that is mapped by redis' crc16 + * to any given redis cluster slot. + * + * The array indexes are slot numbers, so that given a desired slot, this string is guaranteed + * to make redis cluster route a request to the shard holding this slot + */ + +const char *crc16_slot_table[] = { +"06S", "Qi", "5L5", "4Iu", "4gY", "460", "1Y7", "1LV", "0QG", "ru", "7Ok", "4ji", "4DE", "65n", "2JH", "I8", "F9", "SX", "7nF", "4KD", +"4eh", "6PK", "2ke", "1Ng", "0Sv", "4L", "491", "4hX", "4Ft", "5C4", "2Hy", "09R", "021", "0cX", "4Xv", "6mU", "6Cy", "42R", "0Mt", "nF", +"cv", "1Pe", "5kK", "6NI", "74L", "4UF", "0nh", "MZ", "2TJ", "0ai", "4ZG", "6od", "6AH", "40c", "0OE", "lw", "aG", "0Bu", "5iz", "6Lx", +"5R7", "4Ww", "0lY", "Ok", "5n3", "4ks", "8YE", "7g", "2KR", "1nP", "714", "64t", "69D", "4Ho", "07I", "Ps", "2hN", "1ML", "4fC", "7CA", +"avs", "4iB", "0Rl", "5V", "2Ic", "08H", "4Gn", "66E", "aUo", "b4e", "05x", "RB", "8f", "8VD", "4dr", "5a2", "4zp", "6OS", "bl", "355", +"0or", "1j2", "75V", "bno", "4Yl", "6lO", "Ap", "0bB", "0Ln", "2yM", "6Bc", "43H", "4xA", "6Mb", "22D", "14", "0mC", "Nq", "6cN", "4Vm", +"ban", "aDl", "CA", "14Z", "8GG", "mm", "549", "41y", "53t", "464", "1Y3", "1LR", "06W", "Qm", "5L1", "4Iq", "4DA", "65j", "2JL", "1oN", +"0QC", "6y", "7Oo", "4jm", "4el", "6PO", "9x", "1Nc", "04f", "2EM", "7nB", "bqs", "4Fp", "5C0", "d6F", "09V", "0Sr", "4H", "495", "bRo", +"aio", "42V", "0Mp", "nB", "025", "17u", "4Xr", "6mQ", "74H", "4UB", "0nl", "3Kn", "cr", "1Pa", "5kO", "6NM", "6AL", "40g", "0OA", "ls", +"2TN", "0am", "4ZC", "aEr", "5R3", "4Ws", "18t", "Oo", "aC", "0Bq", "bCl", "afn", "2KV", "1nT", "5Uz", "64p", "5n7", "4kw", "0PY", "7c", +"2hJ", "1MH", "4fG", "6Sd", "7mi", "4Hk", "07M", "Pw", "2Ig", "08L", "4Gj", "66A", "7LD", "4iF", "0Rh", "5R", "8b", "1Oy", "4dv", "5a6", +"7oX", "4JZ", "0qt", "RF", "0ov", "LD", "4A9", "4TX", "4zt", "6OW", "bh", "0AZ", "z9", "oX", "6Bg", "43L", "4Yh", "6lK", "At", "0bF", +"0mG", "Nu", "6cJ", "4Vi", "4xE", "6Mf", "2vH", "10", "8GC", "mi", "5p5", "4uu", "5Kx", "4N8", "CE", "1pV", "0QO", "6u", "7Oc", "4ja", +"4DM", "65f", "3Za", "I0", "0rS", "Qa", "68V", "b7F", "4gQ", "468", "dSo", "285", "274", "4D", "499", "4hP", "b8G", "67W", "0h3", "09Z", +"F1", "SP", "7nN", "4KL", "51I", "6PC", "9t", "1No", "21g", "1Pm", "5kC", "6NA", "74D", "4UN", "X3", "MR", "029", "0cP", "bbM", "79t", +"4c3", "42Z", "8Dd", "nN", "aO", "8Ke", "4yS", "4l2", "76u", "635", "0lQ", "Oc", "BS", "W2", "4ZO", "6ol", "7Qa", "40k", "0OM", "2zn", +"69L", "4Hg", "07A", "2Fj", "2hF", "k6", "4fK", "6Sh", "7Ny", "6K9", "0PU", "7o", "2KZ", "1nX", "4EW", "4P6", "7oT", "4JV", "05p", "RJ", +"8n", "1Ou", "4dz", "6QY", "7LH", "4iJ", "d7", "qV", "2Ik", "1li", "4Gf", "66M", "4Yd", "6lG", "Ax", "0bJ", "z5", "oT", "6Bk", "4wH", +"4zx", "aeI", "bd", "0AV", "0oz", "LH", "4A5", "4TT", "5Kt", "4N4", "CI", "14R", "0NW", "me", "541", "41q", "4xI", "6Mj", "22L", "u4", +"0mK", "Ny", "6cF", "4Ve", "4DI", "65b", "2JD", "I4", "0QK", "6q", "7Og", "4je", "4gU", "4r4", "2iX", "1LZ", "0rW", "Qe", "5L9", "4Iy", +"4Fx", "5C8", "0h7", "1mw", "0Sz", "pH", "7MV", "4hT", "4ed", "6PG", "9p", "1Nk", "F5", "ST", "7nJ", "4KH", "7pH", "4UJ", "X7", "MV", +"cz", "1Pi", "5kG", "6NE", "4c7", "4vV", "0Mx", "nJ", "0v5", "0cT", "4Xz", "6mY", "6bX", "5GZ", "0lU", "Og", "aK", "0By", "4yW", "4l6", +"6AD", "40o", "0OI", "2zj", "BW", "W6", "4ZK", "6oh", "2hB", "k2", "4fO", "6Sl", "69H", "4Hc", "07E", "2Fn", "d5e", "83m", "4ES", "4P2", +"a0F", "bQL", "0PQ", "7k", "8j", "1Oq", "50W", "hbv", "7oP", "4JR", "05t", "RN", "2Io", "08D", "4Gb", "66I", "7LL", "4iN", "d3", "5Z", +"z1", "oP", "6Bo", "43D", "5IA", "6lC", "2Wm", "0bN", "8ff", "LL", "4A1", "4TP", "cPn", "aeM", "0T3", "0AR", "0NS", "ma", "545", "41u", +"5Kp", "4N0", "CM", "14V", "0mO", "2Xl", "6cB", "4Va", "4xM", "6Mn", "22H", "18", "04s", "SI", "7nW", "4KU", "4ey", "6PZ", "9m", "1Nv", +"e4", "pU", "7MK", "4hI", "4Fe", "67N", "2Hh", "09C", "06B", "Qx", "68O", "4Id", "4gH", "6Rk", "2iE", "j5", "0QV", "6l", "5o8", "4jx", +"4DT", "4Q5", "2JY", "82j", "BJ", "0ax", "4ZV", "4O7", "552", "40r", "0OT", "lf", "aV", "t7", "4yJ", "6Li", "6bE", "4Wf", "0lH", "Oz", +"2Vj", "0cI", "4Xg", "6mD", "6Ch", "42C", "0Me", "nW", "cg", "1Pt", "5kZ", "6NX", "7pU", "4UW", "0ny", "MK", "7LQ", "4iS", "267", "5G", +"0i0", "08Y", "b9D", "66T", "7oM", "4JO", "G2", "RS", "8w", "1Ol", "4dc", "7Aa", "atS", "4kb", "0PL", "7v", "2KC", "H3", "4EN", "64e", +"69U", "b6E", "07X", "Pb", "dRl", "296", "4fR", "4s3", "4xP", "4m1", "22U", "8Jf", "0mR", "0x3", "77v", "626", "5Km", "6no", "CP", "V1", +"0NN", "3kL", "7Pb", "41h", "4za", "6OB", "20d", "0AO", "Y0", "LQ", "6an", "4TM", "bcN", "78w", "Aa", "0bS", "8Eg", "oM", "4b0", "43Y", +"51T", "azL", "9i", "1Nr", "04w", "SM", "7nS", "4KQ", "4Fa", "67J", "2Hl", "09G", "e0", "4Y", "7MO", "4hM", "4gL", "6Ro", "2iA", "j1", +"06F", "2Gm", "68K", "5YA", "4DP", "4Q1", "d4f", "82n", "0QR", "6h", "a1E", "bPO", "556", "40v", "0OP", "lb", "BN", "15U", "4ZR", "4O3", +"6bA", "4Wb", "0lL", "2Yo", "aR", "t3", "4yN", "6Lm", "6Cl", "42G", "0Ma", "nS", "2Vn", "0cM", "4Xc", "79i", "74Y", "4US", "8ge", "MO", +"cc", "1Pp", "bAL", "adN", "0i4", "1lt", "5WZ", "66P", "7LU", "4iW", "0Ry", "5C", "8s", "1Oh", "4dg", "6QD", "7oI", "4JK", "G6", "RW", +"2KG", "H7", "4EJ", "64a", "7Nd", "4kf", "0PH", "7r", "1X8", "1MY", "4fV", "4s7", "69Q", "4Hz", "0sT", "Pf", "0mV", "Nd", "5S8", "4Vx", +"4xT", "4m5", "22Q", "0Cz", "0NJ", "mx", "7Pf", "41l", "5Ki", "6nk", "CT", "V5", "Y4", "LU", "6aj", "4TI", "4ze", "6OF", "by", "0AK", +"2l9", "oI", "4b4", "4wU", "4Yy", "6lZ", "Ae", "0bW", "0So", "4U", "7MC", "4hA", "4Fm", "67F", "3XA", "09K", "0ps", "SA", "aTl", "b5f", +"4eq", "6PR", "9e", "8WG", "8XF", "6d", "5o0", "4jp", "707", "65w", "1z2", "1oS", "06J", "Qp", "68G", "4Il", "53i", "6Rc", "2iM", "1LO", +"23G", "07", "4yB", "6La", "6bM", "4Wn", "18i", "Or", "BB", "0ap", "c4D", "aEo", "5q2", "40z", "8FD", "ln", "co", "346", "5kR", "6NP", +"74U", "bol", "0nq", "MC", "2Vb", "0cA", "4Xo", "6mL", "7SA", "42K", "0Mm", "2xN", "7oE", "4JG", "05a", "2DJ", "2jf", "1Od", "4dk", "6QH", +"482", "5yz", "0Ru", "5O", "0i8", "08Q", "4Gw", "5B7", "5M6", "4Hv", "07P", "Pj", "1X4", "1MU", "4fZ", "473", "7Nh", "4kj", "0PD", "sv", +"2KK", "1nI", "4EF", "64m", "5Ke", "6ng", "CX", "V9", "0NF", "mt", "7Pj", "4uh", "4xX", "4m9", "1F6", "0Cv", "0mZ", "Nh", "5S4", "4Vt", +"4Yu", "6lV", "Ai", "16r", "0Lw", "oE", "4b8", "43Q", "4zi", "6OJ", "bu", "0AG", "Y8", "LY", "6af", "4TE", "4Fi", "67B", "2Hd", "09O", +"e8", "4Q", "7MG", "4hE", "4eu", "6PV", "9a", "1Nz", "0pw", "SE", "aTh", "4KY", "4DX", "4Q9", "1z6", "1oW", "0QZ", "rh", "5o4", "4jt", +"4gD", "6Rg", "2iI", "j9", "06N", "Qt", "68C", "4Ih", "6bI", "4Wj", "0lD", "Ov", "aZ", "03", "4yF", "6Le", "5q6", "4tv", "0OX", "lj", +"BF", "0at", "4ZZ", "6oy", "74Q", "5Ez", "0nu", "MG", "ck", "1Px", "5kV", "6NT", "6Cd", "42O", "0Mi", "2xJ", "2Vf", "0cE", "4Xk", "6mH", +"2jb", "8VY", "4do", "6QL", "7oA", "4JC", "05e", "2DN", "d7E", "08U", "4Gs", "5B3", "486", "bSl", "0Rq", "5K", "1X0", "1MQ", "52w", "477", +"5M2", "4Hr", "07T", "Pn", "2KO", "1nM", "4EB", "64i", "7Nl", "4kn", "8YX", "7z", "0NB", "mp", "7Pn", "41d", "5Ka", "6nc", "2UM", "14G", +"19w", "Nl", "5S0", "4Vp", "bBo", "agm", "1F2", "0Cr", "0Ls", "oA", "ahl", "43U", "4Yq", "6lR", "Am", "16v", "0oo", "2ZL", "6ab", "4TA", +"4zm", "6ON", "bq", "0AC", "2VY", "0cz", "4XT", "4M5", "570", "42p", "0MV", "nd", "cT", "v5", "5ki", "6Nk", "74n", "4Ud", "0nJ", "Mx", +"By", "0aK", "4Ze", "6oF", "6Aj", "40A", "y4", "lU", "ae", "0BW", "4yy", "581", "4B4", "4WU", "18R", "OI", "06q", "QK", "7lU", "4IW", +"53R", "6RX", "0I4", "1Lt", "g6", "rW", "7OI", "4jK", "4Dg", "65L", "2Jj", "1oh", "0pH", "Sz", "7nd", "4Kf", "4eJ", "6Pi", "2kG", "h7", +"0ST", "4n", "7Mx", "4hz", "4FV", "4S7", "1x8", "09p", "4zR", "4o3", "bN", "8Hd", "0oP", "Lb", "75t", "604", "4YN", "6lm", "AR", "T3", +"0LL", "2yo", "6BA", "43j", "4xc", "agR", "22f", "0CM", "0ma", "NS", "6cl", "4VO", "baL", "aDN", "Cc", "14x", "8Ge", "mO", "7PQ", "4uS", +"7NS", "4kQ", "245", "7E", "0k2", "1nr", "coo", "64V", "69f", "4HM", "E0", "PQ", "2hl", "1Mn", "4fa", "6SB", "7Lb", "5yA", "0RN", "5t", +"2IA", "J1", "4GL", "66g", "aUM", "b4G", "05Z", "0d3", "8D", "8Vf", "4dP", "459", "574", "42t", "0MR", "0X3", "dln", "17W", "4XP", "4M1", +"74j", "5EA", "0nN", "3KL", "cP", "29", "5km", "6No", "6An", "40E", "y0", "lQ", "2Tl", "0aO", "4Za", "6oB", "4B0", "4WQ", "18V", "OM", +"aa", "0BS", "bCN", "585", "53V", "axN", "0I0", "1Lp", "06u", "QO", "68x", "4IS", "4Dc", "65H", "2Jn", "1ol", "g2", "rS", "7OM", "4jO", +"4eN", "6Pm", "9Z", "h3", "04D", "2Eo", "aTS", "4Kb", "4FR", "4S3", "d6d", "09t", "0SP", "4j", "a3G", "bRM", "0oT", "Lf", "6aY", "4Tz", +"4zV", "4o7", "bJ", "0Ax", "0LH", "oz", "6BE", "43n", "4YJ", "6li", "AV", "T7", "0me", "NW", "6ch", "4VK", "4xg", "6MD", "22b", "0CI", +"0Ny", "mK", "7PU", "4uW", "5KZ", "6nX", "Cg", "1pt", "0k6", "1nv", "4Ey", "64R", "7NW", "4kU", "241", "7A", "2hh", "1Mj", "4fe", "6SF", +"69b", "4HI", "E4", "PU", "2IE", "J5", "4GH", "66c", "7Lf", "4id", "0RJ", "5p", "2jY", "8Vb", "4dT", "4q5", "5O8", "4Jx", "0qV", "Rd", +"21E", "25", "5ka", "6Nc", "74f", "4Ul", "0nB", "Mp", "1f2", "0cr", "bbo", "79V", "578", "42x", "395", "nl", "am", "364", "4yq", "589", +"76W", "bmn", "0ls", "OA", "Bq", "0aC", "4Zm", "6oN", "6Ab", "40I", "0Oo", "2zL", "0Qm", "6W", "7OA", "4jC", "4Do", "65D", "2Jb", "82Q", +"06y", "QC", "68t", "b7d", "4gs", "5b3", "dSM", "8UE", "8ZD", "4f", "5m2", "4hr", "725", "67u", "1x0", "09x", "04H", "Sr", "7nl", "4Kn", +"4eB", "6Pa", "9V", "1NM", "4YF", "6le", "AZ", "0bh", "0LD", "ov", "6BI", "43b", "4zZ", "6Oy", "bF", "0At", "0oX", "Lj", "5Q6", "4Tv", +"5KV", "6nT", "Ck", "14p", "0Nu", "mG", "7PY", "41S", "4xk", "6MH", "22n", "0CE", "0mi", "2XJ", "6cd", "4VG", "69n", "4HE", "E8", "PY", +"2hd", "1Mf", "4fi", "6SJ", "ath", "4kY", "0Pw", "7M", "2Kx", "1nz", "4Eu", "6pV", "5O4", "4Jt", "05R", "Rh", "8L", "1OW", "4dX", "451", +"7Lj", "4ih", "0RF", "qt", "2II", "J9", "4GD", "66o", "74b", "4Uh", "0nF", "Mt", "cX", "21", "5ke", "6Ng", "5s4", "4vt", "0MZ", "nh", +"1f6", "0cv", "4XX", "4M9", "4B8", "4WY", "0lw", "OE", "ai", "1Rz", "4yu", "6LV", "6Af", "40M", "y8", "lY", "Bu", "0aG", "4Zi", "6oJ", +"4Dk", "6qH", "2Jf", "1od", "0Qi", "6S", "7OE", "4jG", "4gw", "5b7", "0I8", "1Lx", "0ru", "QG", "68p", "5Yz", "4FZ", "67q", "1x4", "1mU", +"0SX", "4b", "5m6", "4hv", "4eF", "6Pe", "9R", "1NI", "04L", "Sv", "7nh", "4Kj", "8EX", "or", "6BM", "43f", "4YB", "6la", "2WO", "0bl", +"8fD", "Ln", "5Q2", "4Tr", "cPL", "aeo", "bB", "0Ap", "0Nq", "mC", "ajn", "41W", "5KR", "6nP", "Co", "14t", "0mm", "2XN", "77I", "4VC", +"4xo", "6ML", "22j", "0CA", "3xA", "1Mb", "4fm", "6SN", "69j", "4HA", "07g", "2FL", "d5G", "83O", "4Eq", "64Z", "a0d", "bQn", "0Ps", "7I", +"8H", "1OS", "50u", "455", "5O0", "4Jp", "05V", "Rl", "2IM", "08f", "5Wa", "66k", "7Ln", "4il", "0RB", "5x", "Bh", "0aZ", "4Zt", "6oW", +"4a9", "40P", "0Ov", "lD", "at", "0BF", "4yh", "6LK", "6bg", "4WD", "Z9", "OX", "2VH", "U8", "4XE", "6mf", "6CJ", "42a", "0MG", "nu", +"cE", "1PV", "5kx", "4n8", "5P5", "4Uu", "8gC", "Mi", "04Q", "Sk", "5N7", "4Kw", "51r", "442", "9O", "1NT", "0SE", "pw", "7Mi", "4hk", +"4FG", "67l", "2HJ", "09a", "3", "QZ", "68m", "4IF", "4gj", "6RI", "2ig", "1Le", "0Qt", "6N", "7OX", "4jZ", "4Dv", "5A6", "0j9", "1oy", +"4xr", "6MQ", "22w", "377", "0mp", "NB", "77T", "blm", "5KO", "6nM", "Cr", "14i", "0Nl", "3kn", "ajs", "41J", "4zC", "aer", "20F", "36", +"0oA", "Ls", "6aL", "4To", "bcl", "78U", "AC", "0bq", "386", "oo", "5r3", "4ws", "5l1", "4iq", "9Kf", "5e", "1y3", "1lR", "736", "66v", +"7oo", "4Jm", "05K", "Rq", "8U", "1ON", "4dA", "6Qb", "7NB", "bQs", "0Pn", "7T", "2Ka", "1nc", "4El", "64G", "69w", "b6g", "07z", "1v2", +"dRN", "8TF", "4fp", "5c0", "akm", "40T", "0Or", "1J2", "Bl", "15w", "4Zp", "6oS", "6bc", "5Ga", "0ln", "2YM", "ap", "0BB", "4yl", "6LO", +"6CN", "42e", "0MC", "nq", "2VL", "0co", "4XA", "6mb", "5P1", "4Uq", "8gG", "Mm", "cA", "1PR", "bAn", "adl", "51v", "446", "9K", "1NP", +"04U", "So", "5N3", "4Ks", "4FC", "67h", "2HN", "09e", "0SA", "ps", "7Mm", "4ho", "4gn", "6RM", "2ic", "1La", "7", "2GO", "68i", "4IB", +"4Dr", "5A2", "d4D", "82L", "0Qp", "6J", "a1g", "bPm", "0mt", "NF", "6cy", "4VZ", "4xv", "6MU", "0V9", "0CX", "0Nh", "mZ", "7PD", "41N", +"5KK", "6nI", "Cv", "14m", "0oE", "Lw", "6aH", "4Tk", "4zG", "6Od", "20B", "32", "0LY", "ok", "5r7", "4ww", "5Iz", "6lx", "AG", "0bu", +"1y7", "1lV", "4GY", "4R8", "5l5", "4iu", "1Bz", "5a", "8Q", "i8", "4dE", "6Qf", "7ok", "4Ji", "05O", "Ru", "2Ke", "1ng", "4Eh", "64C", +"7NF", "4kD", "f9", "7P", "2hy", "3m9", "4ft", "5c4", "69s", "4HX", "0sv", "PD", "23e", "0BN", "5iA", "6LC", "6bo", "4WL", "Z1", "OP", +"0t3", "0aR", "c4f", "aEM", "4a1", "40X", "8Ff", "lL", "cM", "8Ig", "5kp", "4n0", "74w", "617", "0nS", "Ma", "3Fa", "U0", "4XM", "6mn", +"6CB", "42i", "0MO", "2xl", "0SM", "4w", "7Ma", "4hc", "4FO", "67d", "2HB", "K2", "04Y", "Sc", "aTN", "b5D", "4eS", "4p2", "9G", "8We", +"256", "6F", "7OP", "4jR", "cnl", "65U", "0j1", "1oq", "D3", "QR", "68e", "4IN", "4gb", "6RA", "2io", "1Lm", "5KG", "6nE", "Cz", "14a", +"x7", "mV", "7PH", "41B", "4xz", "592", "0V5", "0CT", "0mx", "NJ", "4C7", "4VV", "4YW", "4L6", "AK", "0by", "0LU", "og", "563", "43s", +"4zK", "6Oh", "bW", "w6", "0oI", "2Zj", "6aD", "4Tg", "7og", "4Je", "05C", "Ry", "2jD", "i4", "4dI", "6Qj", "5l9", "4iy", "0RW", "5m", +"2IX", "08s", "4GU", "4R4", "7mV", "4HT", "07r", "PH", "0H7", "1Mw", "4fx", "5c8", "7NJ", "4kH", "f5", "sT", "2Ki", "1nk", "4Ed", "64O", +"6bk", "4WH", "Z5", "OT", "ax", "0BJ", "4yd", "6LG", "4a5", "4tT", "0Oz", "lH", "Bd", "0aV", "4Zx", "aEI", "5P9", "4Uy", "0nW", "Me", +"cI", "1PZ", "5kt", "4n4", "6CF", "42m", "0MK", "ny", "2VD", "U4", "4XI", "6mj", "4FK", "6sh", "2HF", "K6", "0SI", "4s", "7Me", "4hg", +"4eW", "4p6", "9C", "1NX", "0pU", "Sg", "7ny", "6k9", "4Dz", "65Q", "0j5", "1ou", "0Qx", "6B", "7OT", "4jV", "4gf", "6RE", "2ik", "1Li", +"D7", "QV", "68a", "4IJ", "x3", "mR", "7PL", "41F", "5KC", "6nA", "2Uo", "14e", "19U", "NN", "4C3", "4VR", "bBM", "596", "0V1", "0CP", +"0LQ", "oc", "567", "43w", "4YS", "4L2", "AO", "16T", "0oM", "2Zn", "75i", "4Tc", "4zO", "6Ol", "bS", "w2", "8Y", "i0", "4dM", "6Qn", +"7oc", "4Ja", "05G", "2Dl", "d7g", "08w", "4GQ", "4R0", "a2D", "bSN", "0RS", "5i", "0H3", "1Ms", "52U", "ayM", "7mR", "4HP", "07v", "PL", +"2Km", "1no", "5UA", "64K", "7NN", "4kL", "f1", "7X", "5nw", "4k7", "fJ", "0Ex", "0kT", "Hf", "6eY", "4Pz", "5Mk", "6hi", "EV", "P7", +"0HH", "kz", "6FE", "47n", "48o", "6ID", "26b", "0GI", "0ie", "JW", "6gh", "4RK", "5OZ", "6jX", "Gg", "0dU", "0Jy", "iK", "4d6", "4qW", +"4z4", "4oU", "1DZ", "3A", "Ye", "0zW", "4Ay", "5D9", "6yj", "4LI", "A4", "TU", "zy", "0YK", "4be", "6WF", "6XG", "4md", "0VJ", "1p", +"2ME", "N5", "4CH", "62c", "5K8", "4Nx", "0uV", "Vd", "xH", "8Rb", "5pu", "4u5", "D", "13W", "5Lq", "4I1", "534", "46t", "0IR", "28y", +"gP", "69", "5om", "6Jo", "6dC", "5AA", "0jN", "3OL", "2Pl", "0eO", "aT1", "6kB", "6En", "44E", "98", "hQ", "ea", "0FS", "49u", "abL", +"4F0", "4SQ", "8ag", "KM", "02u", "UO", "4X2", "4MS", "57V", "a8F", "0M0", "0XQ", "c2", "vS", "7KM", "4nO", "5PB", "61H", "2Nn", "1kl", +"00D", "2Ao", "6zA", "4Ob", "4aN", "6Tm", "yR", "l3", "0WP", "0j", "a7G", "58W", "4BR", "4W3", "ZN", "84l", "0kP", "Hb", "71t", "644", +"5ns", "4k3", "fN", "8Ld", "0HL", "29g", "6FA", "47j", "5Mo", "6hm", "ER", "P3", "0ia", "JS", "6gl", "4RO", "48k", "7Ya", "26f", "0GM", +"8Ce", "iO", "4d2", "4qS", "beL", "hYw", "Gc", "0dQ", "Ya", "0zS", "cko", "60V", "4z0", "4oQ", "205", "3E", "2ll", "0YO", "4ba", "6WB", +"6yn", "4LM", "A0", "TQ", "2MA", "N1", "4CL", "62g", "6XC", "59I", "0VN", "1t", "xL", "8Rf", "54y", "419", "aQM", "b0G", "01Z", "3PP", +"530", "46p", "0IV", "jd", "DH", "0gz", "5Lu", "4I5", "6dG", "4Qd", "0jJ", "Ix", "gT", "r5", "5oi", "6Jk", "6Ej", "44A", "0Kg", "hU", +"Fy", "0eK", "5ND", "6kF", "4F4", "4SU", "1xZ", "KI", "ee", "0FW", "49q", "5x9", "57R", "6VX", "0M4", "0XU", "02q", "UK", "4X6", "4MW", +"5PF", "61L", "2Nj", "1kh", "c6", "vW", "7KI", "4nK", "4aJ", "6Ti", "yV", "l7", "0tH", "Wz", "6zE", "4Of", "4BV", "4W7", "ZJ", "0yx", +"0WT", "0n", "6YY", "4lz", "5Mc", "6ha", "2SO", "0fl", "1Xa", "kr", "6FM", "47f", "bDm", "aao", "fB", "0Ep", "8bD", "Hn", "5U2", "4Pr", +"5OR", "5Z3", "Go", "10t", "0Jq", "iC", "ann", "45W", "48g", "6IL", "ds", "0GA", "0im", "3Lo", "73I", "4RC", "6yb", "4LA", "03g", "2BL", +"zq", "0YC", "4bm", "6WN", "a4d", "bUn", "0Ts", "3I", "Ym", "87O", "4Aq", "5D1", "5K0", "4Np", "01V", "Vl", "2nQ", "1KS", "54u", "415", +"6XO", "4ml", "0VB", "1x", "2MM", "0xn", "5Sa", "62k", "gX", "61", "5oe", "6Jg", "6dK", "4Qh", "0jF", "It", "L", "0gv", "5Ly", "4I9", +"5w4", "4rt", "0IZ", "jh", "ei", "1Vz", "5mT", "5x5", "4F8", "4SY", "0hw", "KE", "Fu", "0eG", "5NH", "6kJ", "6Ef", "44M", "90", "hY", +"0Ui", "2S", "7KE", "4nG", "5PJ", "6uH", "Xw", "1kd", "0vu", "UG", "6xx", "790", "4cw", "5f7", "0M8", "0XY", "0WX", "0b", "5i6", "4lv", +"4BZ", "63q", "ZF", "0yt", "00L", "Wv", "6zI", "4Oj", "4aF", "6Te", "yZ", "0Zh", "0HD", "kv", "6FI", "47b", "5Mg", "6he", "EZ", "0fh", +"0kX", "Hj", "5U6", "4Pv", "7N9", "6Ky", "fF", "0Et", "0Ju", "iG", "6Dx", "45S", "5OV", "5Z7", "Gk", "0dY", "0ii", "3Lk", "6gd", "4RG", +"48c", "6IH", "dw", "0GE", "zu", "0YG", "4bi", "6WJ", "6yf", "4LE", "A8", "TY", "Yi", "1jz", "4Au", "5D5", "4z8", "4oY", "0Tw", "3M", +"xD", "1KW", "54q", "411", "5K4", "4Nt", "01R", "Vh", "2MI", "N9", "4CD", "62o", "6XK", "4mh", "0VF", "ut", "6dO", "4Ql", "0jB", "Ip", +"25E", "65", "5oa", "6Jc", "538", "46x", "9Pg", "jl", "H", "0gr", "bfo", "aCm", "72W", "bin", "0hs", "KA", "em", "324", "49y", "5x1", +"6Eb", "44I", "94", "3nm", "Fq", "0eC", "5NL", "6kN", "5PN", "61D", "Xs", "86Q", "0Um", "2W", "7KA", "4nC", "4cs", "5f3", "39W", "8QE", +"02y", "UC", "aRn", "794", "765", "63u", "ZB", "0yp", "9Ne", "0f", "5i2", "4lr", "4aB", "6Ta", "2oO", "0Zl", "00H", "Wr", "6zM", "4On", +"5lW", "5y6", "dj", "0GX", "0it", "JF", "6gy", "4RZ", "5OK", "6jI", "Gv", "0dD", "83", "iZ", "6De", "45N", "5nf", "6Kd", "24B", "72", +"0kE", "Hw", "6eH", "4Pk", "5Mz", "6hx", "EG", "0fu", "0HY", "kk", "5v7", "4sw", "5h5", "4mu", "1Fz", "1a", "2MT", "0xw", "4CY", "4V8", +"7kk", "4Ni", "01O", "Vu", "xY", "m8", "54l", "6Uf", "6Zg", "4oD", "b9", "3P", "Yt", "0zF", "4Ah", "60C", "4Y9", "4LX", "0wv", "TD", +"zh", "0YZ", "4bt", "5g4", "Fl", "11w", "5NQ", "6kS", "aom", "44T", "0Kr", "1N2", "ep", "0FB", "49d", "6HO", "6fc", "5Ca", "0hn", "3Ml", +"U", "0go", "bfr", "6ib", "6GN", "46e", "0IC", "jq", "gA", "0Ds", "bEn", "hyU", "5T1", "4Qq", "8cG", "Im", "00U", "Wo", "5J3", "4Os", +"55v", "406", "yC", "0Zq", "0WA", "ts", "6YL", "4lo", "4BC", "63h", "2LN", "0ym", "02d", "2CO", "6xa", "4MB", "4cn", "6VM", "2mc", "1Ha", +"0Up", "2J", "a5g", "bTm", "5PS", "5E2", "Xn", "86L", "0ip", "JB", "73T", "bhm", "48z", "5y2", "dn", "337", "87", "3on", "6Da", "45J", +"5OO", "6jM", "Gr", "10i", "0kA", "Hs", "6eL", "4Po", "5nb", "aar", "24F", "76", "8AE", "ko", "5v3", "4ss", "bgl", "aBn", "EC", "0fq", +"2MP", "0xs", "776", "62v", "5h1", "4mq", "9Of", "1e", "2nL", "1KN", "54h", "6Ub", "7ko", "4Nm", "01K", "Vq", "Yp", "0zB", "4Al", "60G", +"6Zc", "bUs", "0Tn", "3T", "zl", "8PF", "4bp", "5g0", "aSm", "787", "03z", "1r2", "4e9", "44P", "0Kv", "hD", "Fh", "0eZ", "5NU", "6kW", +"6fg", "4SD", "0hj", "KX", "et", "0FF", "5mI", "6HK", "6GJ", "46a", "0IG", "ju", "Q", "Q8", "5Ld", "6if", "5T5", "4Qu", "1zz", "Ii", +"gE", "0Dw", "5ox", "4j8", "55r", "402", "yG", "0Zu", "00Q", "Wk", "5J7", "4Ow", "4BG", "63l", "2LJ", "0yi", "0WE", "tw", "6YH", "4lk", +"4cj", "6VI", "2mg", "0XD", "0vh", "UZ", "6xe", "4MF", "5PW", "5E6", "Xj", "1ky", "0Ut", "2N", "7KX", "4nZ", "5OC", "6jA", "2Qo", "0dL", +"1ZA", "iR", "6Dm", "45F", "48v", "acO", "db", "0GP", "94M", "JN", "4G3", "4RR", "5Mr", "4H2", "EO", "12T", "0HQ", "kc", "527", "47w", +"5nn", "6Kl", "fS", "s2", "0kM", "3NO", "71i", "4Pc", "7kc", "4Na", "01G", "3PM", "xQ", "m0", "54d", "6Un", "a6D", "59T", "0VS", "1i", +"197", "85o", "4CQ", "4V0", "4Y1", "4LP", "03v", "TL", "0L3", "0YR", "56U", "a9E", "6Zo", "4oL", "b1", "3X", "2Om", "0zN", "5QA", "60K", +"ex", "0FJ", "49l", "6HG", "6fk", "4SH", "0hf", "KT", "Fd", "0eV", "5NY", "aAI", "4e5", "4pT", "0Kz", "hH", "gI", "1TZ", "5ot", "4j4", +"5T9", "4Qy", "0jW", "Ie", "DU", "Q4", "5Lh", "6ij", "6GF", "46m", "0IK", "jy", "0WI", "0s", "6YD", "4lg", "4BK", "6wh", "ZW", "O6", +"0tU", "Wg", "6zX", "6o9", "4aW", "4t6", "yK", "0Zy", "0Ux", "2B", "7KT", "4nV", "bzI", "61Q", "Xf", "1ku", "02l", "UV", "6xi", "4MJ", +"4cf", "6VE", "2mk", "0XH", "0Jd", "iV", "6Di", "45B", "5OG", "6jE", "Gz", "0dH", "0ix", "JJ", "4G7", "4RV", "48r", "6IY", "df", "0GT", +"0HU", "kg", "523", "47s", "5Mv", "4H6", "EK", "0fy", "0kI", "3NK", "6eD", "4Pg", "5nj", "6Kh", "fW", "s6", "xU", "m4", "5ph", "6Uj", +"7kg", "4Ne", "01C", "Vy", "193", "1hZ", "4CU", "4V4", "5h9", "4my", "0VW", "1m", "zd", "0YV", "4bx", "5g8", "4Y5", "4LT", "03r", "TH", +"Yx", "0zJ", "4Ad", "60O", "6Zk", "4oH", "b5", "wT", "6fo", "4SL", "0hb", "KP", "27e", "0FN", "49h", "6HC", "4e1", "44X", "8Bf", "hL", +"0p3", "0eR", "bdO", "aAM", "70w", "657", "0jS", "Ia", "gM", "8Mg", "5op", "4j0", "6GB", "46i", "0IO", "28d", "Y", "Q0", "5Ll", "6in", +"4BO", "63d", "ZS", "O2", "0WM", "0w", "7Ia", "4lc", "4aS", "4t2", "yO", "8Se", "00Y", "Wc", "aPN", "b1D", "bzM", "61U", "Xb", "1kq", +"216", "2F", "7KP", "4nR", "4cb", "6VA", "2mo", "0XL", "02h", "UR", "6xm", "4MN", "5j7", "4ow", "0TY", "3c", "YG", "0zu", "5Qz", "60p", +"6yH", "4Lk", "03M", "Tw", "2lJ", "0Yi", "4bG", "6Wd", "6Xe", "4mF", "0Vh", "1R", "2Mg", "0xD", "4Cj", "62A", "7kX", "4NZ", "0ut", "VF", +"xj", "1Ky", "5pW", "5e6", "5nU", "6KW", "fh", "0EZ", "0kv", "HD", "4E9", "4PX", "5MI", "6hK", "Et", "0fF", "0Hj", "kX", "6Fg", "47L", +"48M", "6If", "dY", "50", "0iG", "Ju", "6gJ", "4Ri", "5Ox", "4J8", "GE", "0dw", "1Zz", "ii", "5t5", "4qu", "02W", "Um", "5H1", "4Mq", +"57t", "424", "2mP", "0Xs", "0UC", "2y", "7Ko", "4nm", "bzr", "61j", "2NL", "1kN", "00f", "2AM", "6zc", "bus", "4al", "6TO", "yp", "0ZB", +"0Wr", "0H", "a7e", "58u", "4Bp", "5G0", "Zl", "84N", "f", "13u", "5LS", "5Y2", "amo", "46V", "0Ip", "jB", "gr", "1Ta", "5oO", "6JM", +"6da", "4QB", "0jl", "3On", "2PN", "0em", "5Nb", "aAr", "6EL", "44g", "0KA", "hs", "eC", "0Fq", "49W", "abn", "5V3", "4Ss", "8aE", "Ko", +"YC", "0zq", "754", "60t", "5j3", "4os", "9Md", "3g", "2lN", "0Ym", "4bC", "7GA", "6yL", "4Lo", "03I", "Ts", "2Mc", "1ha", "4Cn", "62E", +"6Xa", "4mB", "0Vl", "1V", "xn", "8RD", "5pS", "5e2", "aQo", "b0e", "01x", "VB", "0kr", "1n2", "71V", "bjo", "5nQ", "6KS", "fl", "315", +"0Hn", "29E", "6Fc", "47H", "5MM", "6hO", "Ep", "0fB", "0iC", "Jq", "6gN", "4Rm", "48I", "6Ib", "26D", "54", "8CG", "im", "509", "45y", +"ben", "hYU", "GA", "0ds", "4cY", "420", "2mT", "0Xw", "02S", "Ui", "5H5", "4Mu", "5Pd", "61n", "XY", "M8", "0UG", "vu", "7Kk", "4ni", +"4ah", "6TK", "yt", "0ZF", "B9", "WX", "6zg", "4OD", "4Bt", "5G4", "Zh", "0yZ", "0Wv", "0L", "4y9", "4lX", "6Gy", "46R", "0It", "jF", +"b", "0gX", "5LW", "5Y6", "6de", "4QF", "0jh", "IZ", "gv", "0DD", "5oK", "6JI", "6EH", "44c", "0KE", "hw", "2PJ", "0ei", "5Nf", "6kd", +"5V7", "4Sw", "0hY", "Kk", "eG", "0Fu", "49S", "6Hx", "7ia", "4Lc", "03E", "2Bn", "zS", "o2", "4bO", "6Wl", "a4F", "bUL", "0TQ", "3k", +"YO", "87m", "4AS", "4T2", "7kP", "4NR", "01t", "VN", "xb", "1Kq", "54W", "hfv", "6Xm", "4mN", "1FA", "1Z", "2Mo", "0xL", "4Cb", "62I", +"5MA", "6hC", "2Sm", "0fN", "0Hb", "kP", "6Fo", "47D", "bDO", "aaM", "0P3", "0ER", "8bf", "HL", "4E1", "4PP", "5Op", "4J0", "GM", "10V", +"0JS", "ia", "505", "45u", "48E", "6In", "dQ", "58", "0iO", "3LM", "6gB", "4Ra", "0UK", "2q", "7Kg", "4ne", "5Ph", "61b", "XU", "M4", +"0vW", "Ue", "5H9", "4My", "4cU", "4v4", "2mX", "1HZ", "0Wz", "tH", "4y5", "4lT", "4Bx", "5G8", "Zd", "0yV", "B5", "WT", "6zk", "4OH", +"4ad", "6TG", "yx", "0ZJ", "gz", "0DH", "5oG", "6JE", "6di", "4QJ", "0jd", "IV", "n", "0gT", "680", "6iY", "4g7", "4rV", "0Ix", "jJ", +"eK", "0Fy", "5mv", "4h6", "6fX", "5CZ", "0hU", "Kg", "FW", "S6", "5Nj", "6kh", "6ED", "44o", "0KI", "3nK", "zW", "o6", "4bK", "6Wh", +"6yD", "4Lg", "03A", "2Bj", "YK", "0zy", "4AW", "4T6", "6ZX", "6O9", "0TU", "3o", "xf", "1Ku", "54S", "6UY", "7kT", "4NV", "01p", "VJ", +"2Mk", "0xH", "4Cf", "62M", "6Xi", "4mJ", "0Vd", "uV", "0Hf", "kT", "6Fk", "4sH", "5ME", "6hG", "Ex", "0fJ", "0kz", "HH", "4E5", "4PT", +"5nY", "aaI", "fd", "0EV", "0JW", "ie", "501", "45q", "5Ot", "4J4", "GI", "10R", "0iK", "Jy", "6gF", "4Re", "48A", "6Ij", "dU", "q4", +"5Pl", "61f", "XQ", "M0", "0UO", "2u", "7Kc", "4na", "4cQ", "428", "39u", "8Qg", "0vS", "Ua", "aRL", "b3F", "bxO", "63W", "0l3", "0yR", +"234", "0D", "4y1", "4lP", "55I", "6TC", "2om", "0ZN", "B1", "WP", "6zo", "4OL", "6dm", "4QN", "1zA", "IR", "25g", "0DL", "5oC", "6JA", +"4g3", "46Z", "9PE", "jN", "j", "0gP", "684", "aCO", "72u", "675", "0hQ", "Kc", "eO", "8Oe", "5mr", "4h2", "7Ua", "44k", "0KM", "3nO", +"FS", "S2", "5Nn", "6kl", "4x6", "4mW", "0Vy", "1C", "0m4", "0xU", "5SZ", "62P", "7kI", "4NK", "C6", "VW", "2nj", "1Kh", "54N", "6UD", +"6ZE", "4of", "0TH", "3r", "YV", "L7", "4AJ", "60a", "6yY", "4Lz", "0wT", "Tf", "zJ", "0Yx", "4bV", "4w7", "5lu", "4i5", "dH", "0Gz", +"0iV", "Jd", "5W8", "4Rx", "5Oi", "6jk", "GT", "R5", "0JJ", "ix", "6DG", "45l", "5nD", "6KF", "fy", "0EK", "0kg", "HU", "6ej", "4PI", +"5MX", "5X9", "Ee", "0fW", "1XZ", "kI", "4f4", "4sU", "00w", "WM", "4Z0", "4OQ", "55T", "hgu", "ya", "0ZS", "a0", "0Y", "6Yn", "4lM", +"4Ba", "63J", "2Ll", "0yO", "02F", "2Cm", "6xC", "aG0", "4cL", "6Vo", "2mA", "n1", "0UR", "2h", "a5E", "bTO", "5Pq", "4U1", "XL", "86n", +"FN", "11U", "5Ns", "4K3", "516", "44v", "0KP", "hb", "eR", "p3", "49F", "6Hm", "6fA", "4Sb", "0hL", "3MN", "w", "0gM", "5LB", "7ya", +"6Gl", "46G", "0Ia", "jS", "gc", "0DQ", "bEL", "hyw", "4D2", "4QS", "8ce", "IO", "0m0", "0xQ", "byL", "62T", "4x2", "4mS", "227", "1G", +"2nn", "1Kl", "54J", "7Ea", "7kM", "4NO", "C2", "VS", "YR", "L3", "4AN", "60e", "6ZA", "4ob", "0TL", "3v", "zN", "8Pd", "4bR", "4w3", +"aSO", "b2E", "03X", "Tb", "0iR", "3LP", "73v", "666", "48X", "4i1", "dL", "8Nf", "0JN", "3oL", "6DC", "45h", "5Om", "6jo", "GP", "R1", +"0kc", "HQ", "6en", "4PM", "a09", "6KB", "24d", "0EO", "8Ag", "kM", "4f0", "47Y", "697", "aBL", "Ea", "0fS", "4ay", "5d9", "ye", "0ZW", +"00s", "WI", "4Z4", "4OU", "4Be", "63N", "Zy", "0yK", "a4", "tU", "6Yj", "4lI", "4cH", "6Vk", "2mE", "n5", "02B", "Ux", "6xG", "4Md", +"5Pu", "4U5", "XH", "86j", "0UV", "2l", "5k8", "4nx", "512", "44r", "0KT", "hf", "FJ", "0ex", "5Nw", "4K7", "6fE", "4Sf", "0hH", "Kz", +"eV", "p7", "49B", "6Hi", "6Gh", "46C", "0Ie", "jW", "s", "0gI", "5LF", "6iD", "4D6", "4QW", "0jy", "IK", "gg", "0DU", "5oZ", "6JX", +"7kA", "4NC", "01e", "3Po", "xs", "8RY", "54F", "6UL", "a6f", "59v", "0Vq", "1K", "d3E", "85M", "4Cs", "5F3", "5I2", "4Lr", "03T", "Tn", +"zB", "0Yp", "56w", "437", "6ZM", "4on", "1Da", "3z", "2OO", "0zl", "4AB", "60i", "5Oa", "6jc", "2QM", "0dn", "0JB", "ip", "6DO", "45d", +"48T", "acm", "1B2", "0Gr", "94o", "Jl", "5W0", "4Rp", "5MP", "5X1", "Em", "12v", "0Hs", "kA", "all", "47U", "5nL", "6KN", "fq", "0EC", +"0ko", "3Nm", "6eb", "4PA", "a8", "0Q", "6Yf", "4lE", "4Bi", "63B", "Zu", "0yG", "0tw", "WE", "4Z8", "4OY", "4au", "5d5", "yi", "1Jz", +"0UZ", "vh", "5k4", "4nt", "5Py", "4U9", "XD", "1kW", "02N", "Ut", "6xK", "4Mh", "4cD", "6Vg", "2mI", "n9", "eZ", "43", "49N", "6He", +"6fI", "4Sj", "0hD", "Kv", "FF", "0et", "7n9", "6ky", "5u6", "4pv", "0KX", "hj", "gk", "0DY", "5oV", "5z7", "6dx", "5Az", "0ju", "IG", +"Dw", "0gE", "5LJ", "6iH", "6Gd", "46O", "0Ii", "28B", "xw", "1Kd", "54B", "6UH", "7kE", "4NG", "01a", "3Pk", "0m8", "0xY", "4Cw", "5F7", +"6Xx", "59r", "0Vu", "1O", "zF", "0Yt", "4bZ", "433", "5I6", "4Lv", "03P", "Tj", "YZ", "0zh", "4AF", "60m", "6ZI", "4oj", "0TD", "wv", +"0JF", "it", "6DK", "4qh", "5Oe", "6jg", "GX", "R9", "0iZ", "Jh", "5W4", "4Rt", "48P", "4i9", "dD", "0Gv", "0Hw", "kE", "4f8", "47Q", +"5MT", "5X5", "Ei", "12r", "0kk", "HY", "6ef", "4PE", "5nH", "6KJ", "fu", "0EG", "4Bm", "63F", "Zq", "0yC", "0Wo", "0U", "6Yb", "4lA", +"4aq", "5d1", "ym", "8SG", "0ts", "WA", "aPl", "b1f", "747", "61w", "2NQ", "1kS", "9Lg", "2d", "5k0", "4np", "57i", "6Vc", "2mM", "0Xn", +"02J", "Up", "6xO", "4Ml", "6fM", "4Sn", "1xa", "Kr", "27G", "47", "49J", "6Ha", "5u2", "44z", "8BD", "hn", "FB", "0ep", "bdm", "aAo", +"70U", "bkl", "0jq", "IC", "go", "306", "5oR", "5z3", "7WA", "46K", "0Im", "28F", "Ds", "0gA", "5LN", "6iL", "0cY", "020", "6mT", "4Xw", +"42S", "6Cx", "nG", "0Mu", "1Pd", "cw", "6NH", "5kJ", "4UG", "74M", "3Kk", "0ni", "0ah", "BZ", "6oe", "4ZF", "40b", "6AI", "lv", "0OD", +"0Bt", "aF", "6Ly", "4yZ", "4Wv", "5R6", "Oj", "0lX", "Qh", "06R", "4It", "5L4", "461", "4gX", "1LW", "1Y6", "rt", "0QF", "4jh", "7Oj", +"65o", "4DD", "I9", "2JI", "SY", "F8", "4KE", "7nG", "6PJ", "4ei", "1Nf", "2kd", "4M", "0Sw", "4hY", "490", "5C5", "4Fu", "09S", "2Hx", +"6OR", "4zq", "354", "bm", "LA", "0os", "bnn", "75W", "6lN", "4Ym", "0bC", "Aq", "2yL", "0Lo", "43I", "6Bb", "6Mc", "5ha", "15", "22E", +"Np", "0mB", "4Vl", "6cO", "aDm", "bao", "1pS", "1e2", "ml", "8GF", "41x", "548", "4kr", "5n2", "7f", "8YD", "1nQ", "2KS", "64u", "715", +"4Hn", "69E", "Pr", "07H", "1MM", "2hO", "6Sa", "4fB", "4iC", "7LA", "5W", "0Rm", "08I", "2Ib", "66D", "4Go", "b4d", "aUn", "RC", "05y", +"8VE", "8g", "5a3", "4ds", "42W", "ain", "nC", "0Mq", "17t", "024", "6mP", "4Xs", "4UC", "74I", "3Ko", "0nm", "8IY", "cs", "6NL", "5kN", +"40f", "6AM", "lr", "8FX", "0al", "2TO", "6oa", "4ZB", "4Wr", "5R2", "On", "18u", "0Bp", "aB", "afo", "bCm", "465", "53u", "1LS", "1Y2", +"Ql", "06V", "4Ip", "5L0", "65k", "5Ta", "1oO", "2JM", "6x", "0QB", "4jl", "7On", "6PN", "4em", "1Nb", "9y", "2EL", "04g", "4KA", "7nC", +"5C1", "4Fq", "09W", "d6G", "4I", "0Ss", "bRn", "494", "LE", "0ow", "4TY", "4A8", "6OV", "4zu", "1Qz", "bi", "oY", "z8", "43M", "6Bf", +"6lJ", "4Yi", "0bG", "Au", "Nt", "0mF", "4Vh", "6cK", "6Mg", "4xD", "11", "22A", "mh", "0NZ", "4ut", "5p4", "4N9", "5Ky", "1pW", "CD", +"1nU", "2KW", "64q", "4EZ", "4kv", "5n6", "7b", "0PX", "1MI", "2hK", "6Se", "4fF", "4Hj", "69A", "Pv", "07L", "08M", "2If", "6rH", "4Gk", +"4iG", "7LE", "5S", "0Ri", "1Ox", "8c", "5a7", "4dw", "5Zz", "7oY", "RG", "0qu", "1Pl", "21f", "adR", "5kB", "4UO", "74E", "MS", "X2", +"0cQ", "028", "79u", "bbL", "4vS", "4c2", "nO", "8De", "8Kd", "aN", "4l3", "4yR", "634", "76t", "Ob", "0lP", "W3", "BR", "6om", "4ZN", +"40j", "6AA", "2zo", "0OL", "6t", "0QN", "5zA", "7Ob", "65g", "4DL", "I1", "2JA", "0g3", "06Z", "b7G", "68W", "469", "4gP", "284", "dSn", +"4E", "275", "4hQ", "498", "67V", "b8F", "1mr", "0h2", "SQ", "F0", "4KM", "7nO", "6PB", "4ea", "1Nn", "9u", "6lF", "4Ye", "0bK", "Ay", +"oU", "z4", "43A", "6Bj", "6OZ", "4zy", "0AW", "be", "LI", "2O9", "4TU", "4A4", "4N5", "5Ku", "14S", "CH", "md", "0NV", "41p", "540", +"6Mk", "4xH", "u5", "22M", "Nx", "0mJ", "4Vd", "6cG", "4Hf", "69M", "Pz", "0sH", "k7", "2hG", "6Si", "4fJ", "4kz", "7Nx", "7n", "0PT", +"1nY", "dqh", "4P7", "4EV", "4JW", "7oU", "RK", "05q", "1Ot", "8o", "6QX", "50R", "4iK", "7LI", "qW", "d6", "08A", "2Ij", "66L", "4Gg", +"4UK", "74A", "MW", "X6", "1Ph", "21b", "6ND", "5kF", "4vW", "4c6", "nK", "0My", "0cU", "0v4", "6mX", "5HZ", "4Wz", "6bY", "Of", "0lT", +"0Bx", "aJ", "4l7", "4yV", "40n", "6AE", "lz", "0OH", "W7", "BV", "6oi", "4ZJ", "65c", "4DH", "I5", "2JE", "6p", "0QJ", "4jd", "7Of", +"4r5", "4gT", "280", "2iY", "Qd", "0rV", "4Ix", "5L8", "5C9", "4Fy", "1mv", "0h6", "4A", "1CZ", "4hU", "7MW", "6PF", "4ee", "1Nj", "9q", +"SU", "F4", "4KI", "7nK", "oQ", "z0", "43E", "6Bn", "6lB", "4Ya", "0bO", "2Wl", "LM", "8fg", "4TQ", "4A0", "aeL", "cPo", "0AS", "ba", +"3kP", "0NR", "41t", "544", "4N1", "5Kq", "14W", "CL", "2Xm", "0mN", "5FA", "6cC", "6Mo", "4xL", "19", "22I", "k3", "2hC", "6Sm", "4fN", +"4Hb", "69I", "2Fo", "07D", "83l", "d5d", "4P3", "4ER", "bQM", "a0G", "7j", "0PP", "1Op", "8k", "hbw", "50V", "4JS", "7oQ", "RO", "05u", +"08E", "2In", "66H", "4Gc", "4iO", "7LM", "qS", "d2", "0ay", "BK", "4O6", "4ZW", "40s", "553", "lg", "0OU", "t6", "aW", "6Lh", "4yK", +"4Wg", "6bD", "2Yj", "0lI", "0cH", "2Vk", "6mE", "4Xf", "42B", "6Ci", "nV", "0Md", "1Pu", "cf", "6NY", "bAI", "4UV", "7pT", "MJ", "0nx", +"SH", "04r", "4KT", "7nV", "azI", "4ex", "1Nw", "9l", "pT", "e5", "4hH", "7MJ", "67O", "4Fd", "09B", "2Hi", "Qy", "06C", "4Ie", "68N", +"6Rj", "4gI", "j4", "2iD", "6m", "0QW", "4jy", "5o9", "4Q4", "4DU", "1oZ", "2JX", "4m0", "4xQ", "8Jg", "22T", "Na", "0mS", "627", "77w", +"6nn", "5Kl", "V0", "CQ", "3kM", "0NO", "41i", "7Pc", "6OC", "5jA", "0AN", "20e", "LP", "Y1", "4TL", "6ao", "78v", "bcO", "0bR", "0w3", +"oL", "8Ef", "43X", "4b1", "4iR", "7LP", "5F", "266", "08X", "0i1", "66U", "b9E", "4JN", "7oL", "RR", "G3", "1Om", "8v", "6QA", "4db", +"4kc", "7Na", "7w", "0PM", "H2", "2KB", "64d", "4EO", "b6D", "69T", "Pc", "07Y", "297", "dRm", "4s2", "4fS", "40w", "557", "lc", "0OQ", +"15T", "BO", "4O2", "4ZS", "4Wc", "76i", "2Yn", "0lM", "t2", "aS", "6Ll", "4yO", "42F", "6Cm", "nR", "8Dx", "0cL", "2Vo", "6mA", "4Xb", +"4UR", "74X", "MN", "8gd", "1Pq", "cb", "adO", "bAM", "azM", "51U", "1Ns", "9h", "SL", "04v", "4KP", "7nR", "67K", "5VA", "09F", "2Hm", +"4X", "e1", "4hL", "7MN", "6Rn", "4gM", "j0", "3ya", "2Gl", "06G", "4Ia", "68J", "4Q0", "4DQ", "82o", "d4g", "6i", "0QS", "bPN", "a1D", +"Ne", "0mW", "4Vy", "5S9", "4m4", "4xU", "1SZ", "22P", "my", "0NK", "41m", "7Pg", "6nj", "5Kh", "V4", "CU", "LT", "Y5", "4TH", "6ak", +"6OG", "4zd", "0AJ", "bx", "oH", "0Lz", "4wT", "4b5", "78r", "4Yx", "0bV", "Ad", "1lu", "0i5", "66Q", "4Gz", "4iV", "7LT", "5B", "0Rx", +"1Oi", "8r", "6QE", "4df", "4JJ", "7oH", "RV", "G7", "H6", "2KF", "6ph", "4EK", "4kg", "7Ne", "7s", "0PI", "1MX", "1X9", "4s6", "4fW", +"5XZ", "69P", "Pg", "0sU", "06", "23F", "afr", "4yC", "4Wo", "6bL", "Os", "0lA", "0aq", "BC", "aEn", "c4E", "4ts", "5q3", "lo", "8FE", +"347", "cn", "6NQ", "5kS", "bom", "74T", "MB", "0np", "17i", "2Vc", "6mM", "4Xn", "42J", "6Ca", "2xO", "0Ml", "4T", "0Sn", "5xa", "7MB", +"67G", "4Fl", "09J", "2Ha", "1u2", "04z", "b5g", "aTm", "6PS", "4ep", "8WF", "9d", "6e", "8XG", "4jq", "5o1", "65v", "706", "1oR", "1z3", +"Qq", "06K", "4Im", "68F", "6Rb", "4gA", "1LN", "2iL", "6nf", "5Kd", "V8", "CY", "mu", "0NG", "41a", "7Pk", "4m8", "4xY", "0Cw", "1F7", +"Ni", "19r", "4Vu", "5S5", "6lW", "4Yt", "0bZ", "Ah", "oD", "0Lv", "43P", "4b9", "6OK", "4zh", "0AF", "bt", "LX", "Y9", "4TD", "6ag", +"4JF", "7oD", "RZ", "0qh", "1Oe", "2jg", "6QI", "4dj", "4iZ", "483", "5N", "0Rt", "08P", "0i9", "5B6", "4Gv", "4Hw", "5M7", "Pk", "07Q", +"1MT", "1X5", "472", "52r", "4kk", "7Ni", "sw", "0PE", "1nH", "2KJ", "64l", "4EG", "4Wk", "6bH", "Ow", "0lE", "02", "23B", "6Ld", "4yG", +"4tw", "5q7", "lk", "0OY", "0au", "BG", "6ox", "5Jz", "4UZ", "74P", "MF", "0nt", "1Py", "cj", "6NU", "5kW", "42N", "6Ce", "nZ", "0Mh", +"0cD", "2Vg", "6mI", "4Xj", "67C", "4Fh", "09N", "2He", "4P", "e9", "4hD", "7MF", "6PW", "4et", "3n9", "2ky", "SD", "0pv", "4KX", "7nZ", +"4Q8", "4DY", "1oV", "1z7", "6a", "1Az", "4ju", "5o5", "6Rf", "4gE", "j8", "2iH", "Qu", "06O", "4Ii", "68B", "mq", "0NC", "41e", "7Po", +"6nb", "bar", "14F", "2UL", "Nm", "19v", "4Vq", "5S1", "agl", "bBn", "0Cs", "1F3", "1I2", "0Lr", "43T", "ahm", "6lS", "4Yp", "16w", "Al", +"2ZM", "0on", "5Da", "6ac", "6OO", "4zl", "0AB", "bp", "1Oa", "8z", "6QM", "4dn", "4JB", "aUs", "2DO", "05d", "08T", "d7D", "5B2", "4Gr", +"bSm", "487", "5J", "0Rp", "1MP", "1X1", "476", "52v", "4Hs", "5M3", "Po", "07U", "1nL", "2KN", "64h", "4EC", "4ko", "7Nm", "ss", "0PA", +"QJ", "06p", "4IV", "7lT", "6RY", "4gz", "1Lu", "0I5", "rV", "g7", "4jJ", "7OH", "65M", "4Df", "1oi", "2Jk", "2Ej", "04A", "4Kg", "7ne", +"6Ph", "4eK", "h6", "2kF", "4o", "0SU", "5xZ", "7My", "4S6", "4FW", "09q", "1x9", "17R", "2VX", "4M4", "4XU", "42q", "571", "ne", "0MW", +"v4", "cU", "6Nj", "5kh", "4Ue", "74o", "My", "0nK", "0aJ", "Bx", "6oG", "4Zd", "4tH", "6Ak", "lT", "y5", "0BV", "ad", "580", "4yx", +"4WT", "4B5", "OH", "0lz", "4kP", "7NR", "7D", "244", "1ns", "0k3", "64W", "con", "4HL", "69g", "PP", "E1", "1Mo", "2hm", "6SC", "52I", +"4ia", "7Lc", "5u", "0RO", "J0", "3Ya", "66f", "4GM", "b4F", "aUL", "Ra", "0qS", "8Vg", "8E", "458", "4dQ", "4o2", "4zS", "8He", "bO", +"Lc", "0oQ", "605", "75u", "6ll", "4YO", "T2", "AS", "2yn", "0LM", "43k", "7Ra", "6MA", "4xb", "0CL", "22g", "NR", "19I", "4VN", "6cm", +"aDO", "baM", "14y", "Cb", "mN", "8Gd", "41Z", "7PP", "axO", "53W", "1Lq", "0I1", "QN", "06t", "4IR", "68y", "65I", "4Db", "1om", "2Jo", +"6Z", "g3", "4jN", "7OL", "6Pl", "4eO", "h2", "2kB", "2En", "04E", "4Kc", "7na", "4S2", "4FS", "09u", "d6e", "4k", "0SQ", "bRL", "a3F", +"42u", "575", "na", "0MS", "17V", "dlo", "4M0", "4XQ", "4Ua", "74k", "3KM", "0nO", "28", "cQ", "6Nn", "5kl", "40D", "6Ao", "lP", "y1", +"0aN", "2Tm", "6oC", "5JA", "4WP", "4B1", "OL", "18W", "0BR", "0W3", "584", "bCO", "1nw", "0k7", "64S", "4Ex", "4kT", "7NV", "sH", "0Pz", +"1Mk", "2hi", "6SG", "4fd", "4HH", "69c", "PT", "E5", "J4", "2ID", "66b", "4GI", "4ie", "7Lg", "5q", "0RK", "1OZ", "8A", "4q4", "4dU", +"4Jy", "5O9", "Re", "0qW", "Lg", "0oU", "5DZ", "6aX", "4o6", "4zW", "0Ay", "bK", "2yj", "0LI", "43o", "6BD", "6lh", "4YK", "T6", "AW", +"NV", "0md", "4VJ", "6ci", "6ME", "4xf", "0CH", "22c", "mJ", "0Nx", "4uV", "7PT", "6nY", "baI", "1pu", "Cf", "6V", "0Ql", "4jB", "aus", +"65E", "4Dn", "1oa", "2Jc", "QB", "06x", "b7e", "68u", "5b2", "4gr", "8UD", "dSL", "4g", "8ZE", "4hs", "5m3", "67t", "724", "09y", "1x1", +"Ss", "04I", "4Ko", "7nm", "azr", "4eC", "1NL", "9W", "24", "21D", "6Nb", "bAr", "4Um", "74g", "Mq", "0nC", "0cs", "1f3", "79W", "bbn", +"42y", "579", "nm", "394", "365", "al", "588", "4yp", "bmo", "76V", "1i2", "0lr", "0aB", "Bp", "6oO", "4Zl", "40H", "6Ac", "2zM", "0On", +"4HD", "69o", "PX", "E9", "1Mg", "2he", "6SK", "4fh", "4kX", "7NZ", "7L", "0Pv", "3N9", "2Ky", "6pW", "4Et", "4Ju", "5O5", "Ri", "05S", +"1OV", "8M", "450", "4dY", "4ii", "7Lk", "qu", "0RG", "J8", "2IH", "66n", "4GE", "6ld", "4YG", "0bi", "2WJ", "ow", "0LE", "43c", "6BH", +"6Ox", "5jz", "0Au", "bG", "Lk", "0oY", "4Tw", "5Q7", "6nU", "5KW", "14q", "Cj", "mF", "0Nt", "41R", "7PX", "6MI", "4xj", "0CD", "22o", +"NZ", "0mh", "4VF", "6ce", "65A", "4Dj", "1oe", "2Jg", "6R", "0Qh", "4jF", "7OD", "5b6", "4gv", "1Ly", "0I9", "QF", "0rt", "4IZ", "68q", +"67p", "5Vz", "1mT", "1x5", "4c", "0SY", "4hw", "5m7", "6Pd", "4eG", "1NH", "9S", "Sw", "04M", "4Kk", "7ni", "4Ui", "74c", "Mu", "0nG", +"20", "cY", "6Nf", "5kd", "4vu", "5s5", "ni", "390", "0cw", "1f7", "4M8", "4XY", "4WX", "4B9", "OD", "0lv", "0BZ", "ah", "6LW", "4yt", +"40L", "6Ag", "lX", "y9", "0aF", "Bt", "6oK", "4Zh", "1Mc", "2ha", "6SO", "4fl", "5Xa", "69k", "2FM", "07f", "83N", "d5F", "6pS", "4Ep", +"bQo", "a0e", "7H", "0Pr", "1OR", "8I", "454", "50t", "4Jq", "5O1", "Rm", "05W", "08g", "2IL", "66j", "4GA", "4im", "7Lo", "5y", "0RC", +"os", "0LA", "43g", "6BL", "78I", "4YC", "0bm", "2WN", "Lo", "8fE", "4Ts", "5Q3", "aen", "cPM", "0Aq", "bC", "mB", "0Np", "41V", "ajo", +"6nQ", "5KS", "14u", "Cn", "2XO", "0ml", "4VB", "6ca", "6MM", "4xn", "1Sa", "22k", "Sj", "04P", "4Kv", "5N6", "443", "4eZ", "1NU", "9N", +"pv", "0SD", "4hj", "7Mh", "67m", "4FF", "1mI", "2HK", "2GJ", "2", "4IG", "68l", "6RH", "4gk", "1Ld", "2if", "6O", "0Qu", "5zz", "7OY", +"5A7", "4Dw", "1ox", "0j8", "15r", "Bi", "6oV", "4Zu", "40Q", "4a8", "lE", "0Ow", "0BG", "au", "6LJ", "4yi", "4WE", "6bf", "OY", "Z8", +"U9", "2VI", "6mg", "4XD", "4vh", "6CK", "nt", "0MF", "1PW", "cD", "4n9", "5ky", "4Ut", "5P4", "Mh", "0nZ", "4ip", "5l0", "5d", "9Kg", +"08z", "1y2", "66w", "737", "4Jl", "7on", "Rp", "05J", "1OO", "8T", "6Qc", "50i", "4kA", "7NC", "7U", "0Po", "1nb", "dqS", "64F", "4Em", +"b6f", "69v", "PA", "0ss", "8TG", "dRO", "5c1", "4fq", "6MP", "4xs", "376", "22v", "NC", "0mq", "bll", "77U", "6nL", "5KN", "14h", "Cs", +"3ko", "0Nm", "41K", "7PA", "6Oa", "4zB", "37", "20G", "Lr", "8fX", "4Tn", "6aM", "78T", "bcm", "0bp", "AB", "on", "387", "43z", "5r2", +"447", "51w", "1NQ", "9J", "Sn", "04T", "4Kr", "5N2", "67i", "4FB", "09d", "2HO", "4z", "1Ca", "4hn", "7Ml", "6RL", "4go", "8UY", "2ib", +"2GN", "6", "4IC", "68h", "5A3", "4Ds", "82M", "d4E", "6K", "0Qq", "bPl", "a1f", "40U", "akl", "lA", "0Os", "15v", "Bm", "6oR", "4Zq", +"4WA", "6bb", "2YL", "0lo", "0BC", "aq", "6LN", "4ym", "42d", "6CO", "np", "0MB", "0cn", "2VM", "6mc", "5Ha", "4Up", "5P0", "Ml", "8gF", +"1PS", "1E2", "adm", "bAo", "1lW", "1y6", "4R9", "4GX", "4it", "5l4", "qh", "0RZ", "i9", "8P", "6Qg", "4dD", "4Jh", "7oj", "Rt", "05N", +"1nf", "2Kd", "64B", "4Ei", "4kE", "7NG", "7Q", "f8", "1Mz", "2hx", "5c5", "4fu", "4HY", "69r", "PE", "0sw", "NG", "0mu", "5Fz", "6cx", +"6MT", "4xw", "0CY", "0V8", "3kk", "0Ni", "41O", "7PE", "6nH", "5KJ", "14l", "Cw", "Lv", "0oD", "4Tj", "6aI", "6Oe", "4zF", "33", "bZ", +"oj", "0LX", "4wv", "5r6", "6ly", "4YZ", "0bt", "AF", "4v", "0SL", "4hb", "awS", "67e", "4FN", "K3", "2HC", "Sb", "04X", "b5E", "aTO", +"4p3", "4eR", "8Wd", "9F", "6G", "257", "4jS", "7OQ", "65T", "cnm", "1op", "0j0", "QS", "D2", "4IO", "68d", "7Ba", "4gc", "1Ll", "2in", +"0BO", "23d", "6LB", "4ya", "4WM", "6bn", "OQ", "Z0", "0aS", "Ba", "aEL", "c4g", "40Y", "4a0", "lM", "8Fg", "8If", "cL", "4n1", "5kq", +"616", "74v", "3KP", "0nR", "U1", "2VA", "6mo", "4XL", "42h", "6CC", "2xm", "0MN", "4Jd", "7of", "Rx", "05B", "i5", "2jE", "6Qk", "4dH", +"4ix", "5l8", "5l", "0RV", "08r", "2IY", "4R5", "4GT", "4HU", "7mW", "PI", "07s", "1Mv", "0H6", "5c9", "4fy", "4kI", "7NK", "sU", "f4", +"1nj", "2Kh", "64N", "4Ee", "6nD", "5KF", "1ph", "2Uj", "mW", "x6", "41C", "7PI", "593", "5hZ", "0CU", "0V4", "NK", "0my", "4VW", "4C6", +"4L7", "4YV", "0bx", "AJ", "of", "0LT", "43r", "562", "6Oi", "4zJ", "w7", "bV", "Lz", "0oH", "4Tf", "6aE", "67a", "4FJ", "K7", "2HG", +"4r", "0SH", "4hf", "7Md", "4p7", "4eV", "1NY", "9B", "Sf", "0pT", "4Kz", "7nx", "65P", "5TZ", "1ot", "0j4", "6C", "0Qy", "4jW", "7OU", +"6RD", "4gg", "1Lh", "2ij", "QW", "D6", "4IK", "7lI", "4WI", "6bj", "OU", "Z4", "0BK", "ay", "6LF", "4ye", "4tU", "4a4", "lI", "2o9", +"0aW", "Be", "6oZ", "4Zy", "4Ux", "5P8", "Md", "0nV", "8Ib", "cH", "4n5", "5ku", "42l", "6CG", "nx", "0MJ", "U5", "2VE", "6mk", "4XH", +"i1", "8X", "6Qo", "4dL", "5ZA", "7ob", "2Dm", "05F", "08v", "d7f", "4R1", "4GP", "bSO", "a2E", "5h", "0RR", "1Mr", "0H2", "ayL", "52T", +"4HQ", "69z", "PM", "07w", "1nn", "2Kl", "64J", "4Ea", "4kM", "7NO", "7Y", "f0", "mS", "x2", "41G", "7PM", "aDR", "5KB", "14d", "2Un", +"NO", "19T", "4VS", "4C2", "597", "bBL", "0CQ", "0V0", "ob", "0LP", "43v", "566", "4L3", "4YR", "16U", "AN", "2Zo", "0oL", "4Tb", "6aA", +"6Om", "4zN", "w3", "bR", "4oT", "4z5", "wH", "0Tz", "0zV", "Yd", "5D8", "4Ax", "4LH", "6yk", "TT", "A5", "0YJ", "zx", "6WG", "4bd", +"4me", "6XF", "1q", "0VK", "N4", "2MD", "62b", "4CI", "4Ny", "5K9", "Ve", "0uW", "1KZ", "xI", "4u4", "5pt", "4k6", "5nv", "0Ey", "fK", +"Hg", "0kU", "641", "6eX", "6hh", "5Mj", "P6", "EW", "29b", "0HI", "47o", "6FD", "6IE", "48n", "0GH", "dz", "JV", "0id", "4RJ", "6gi", +"6jY", "beI", "0dT", "Gf", "iJ", "0Jx", "4qV", "4d7", "UN", "02t", "4MR", "4X3", "a8G", "57W", "0XP", "0M1", "2Z", "c3", "4nN", "7KL", +"61I", "5PC", "1km", "2No", "2An", "00E", "4Oc", "7ja", "6Tl", "4aO", "l2", "yS", "0k", "0WQ", "58V", "a7F", "4W2", "4BS", "84m", "ZO", +"13V", "E", "4I0", "5Lp", "46u", "535", "ja", "0IS", "68", "gQ", "6Jn", "5ol", "4Qa", "6dB", "3OM", "0jO", "0eN", "2Pm", "6kC", "5NA", +"44D", "6Eo", "hP", "99", "0FR", "0S3", "abM", "49t", "4SP", "4F1", "KL", "8af", "0zR", "0o3", "60W", "ckn", "4oP", "4z1", "3D", "204", +"0YN", "2lm", "6WC", "56I", "4LL", "6yo", "TP", "A1", "N0", "903", "62f", "4CM", "4ma", "6XB", "1u", "0VO", "8Rg", "xM", "418", "54x", +"b0F", "aQL", "Va", "0uS", "Hc", "0kQ", "645", "71u", "4k2", "5nr", "8Le", "fO", "29f", "0HM", "47k", "7Va", "6hl", "5Mn", "P2", "ES", +"JR", "1yA", "4RN", "6gm", "6IA", "48j", "0GL", "26g", "iN", "8Cd", "45Z", "4d3", "hYv", "beM", "0dP", "Gb", "6VY", "4cz", "0XT", "0M5", +"UJ", "02p", "4MV", "4X7", "61M", "5PG", "1ki", "Xz", "vV", "c7", "4nJ", "7KH", "6Th", "4aK", "l6", "yW", "2Aj", "00A", "4Og", "6zD", +"4W6", "4BW", "0yy", "ZK", "0o", "0WU", "58R", "6YX", "46q", "531", "je", "0IW", "13R", "A", "4I4", "5Lt", "4Qe", "6dF", "Iy", "0jK", +"r4", "gU", "6Jj", "5oh", "4pH", "6Ek", "hT", "0Kf", "0eJ", "Fx", "6kG", "5NE", "4ST", "4F5", "KH", "0hz", "0FV", "ed", "5x8", "49p", +"bvs", "6yc", "2BM", "03f", "0YB", "zp", "6WO", "4bl", "bUo", "a4e", "3H", "0Tr", "87N", "Yl", "5D0", "4Ap", "4Nq", "5K1", "Vm", "01W", +"1KR", "xA", "414", "54t", "4mm", "6XN", "1y", "0VC", "0xo", "2ML", "62j", "4CA", "7xA", "5Mb", "0fm", "2SN", "ks", "0HA", "47g", "6FL", +"aan", "bDl", "0Eq", "fC", "Ho", "8bE", "4Ps", "5U3", "5Z2", "5OS", "10u", "Gn", "iB", "0Jp", "45V", "ano", "6IM", "48f", "1Wa", "dr", +"3Ln", "0il", "4RB", "6ga", "2R", "0Uh", "4nF", "7KD", "61A", "5PK", "1ke", "Xv", "UF", "0vt", "4MZ", "6xy", "5f6", "4cv", "0XX", "0M9", +"0c", "0WY", "4lw", "5i7", "63p", "5Rz", "0yu", "ZG", "Ww", "00M", "4Ok", "6zH", "6Td", "4aG", "0Zi", "2oJ", "60", "gY", "6Jf", "5od", +"4Qi", "6dJ", "Iu", "0jG", "0gw", "M", "4I8", "5Lx", "4ru", "5w5", "ji", "1Yz", "0FZ", "eh", "5x4", "5mU", "4SX", "4F9", "KD", "0hv", +"0eF", "Ft", "6kK", "5NI", "44L", "6Eg", "hX", "91", "0YF", "zt", "6WK", "4bh", "4LD", "6yg", "TX", "A9", "0zZ", "Yh", "5D4", "4At", +"4oX", "4z9", "3L", "0Tv", "1KV", "xE", "410", "54p", "4Nu", "5K5", "Vi", "01S", "N8", "2MH", "62n", "4CE", "4mi", "6XJ", "uu", "0VG", +"kw", "0HE", "47c", "6FH", "6hd", "5Mf", "0fi", "2SJ", "Hk", "0kY", "4Pw", "5U7", "6Kx", "5nz", "0Eu", "fG", "iF", "0Jt", "45R", "6Dy", +"5Z6", "5OW", "0dX", "Gj", "JZ", "0ih", "4RF", "6ge", "6II", "48b", "0GD", "dv", "61E", "5PO", "1ka", "Xr", "2V", "0Ul", "4nB", "aqs", +"5f2", "4cr", "8QD", "39V", "UB", "02x", "795", "aRo", "63t", "764", "0yq", "ZC", "0g", "9Nd", "4ls", "5i3", "7DA", "4aC", "0Zm", "2oN", +"Ws", "00I", "4Oo", "6zL", "4Qm", "6dN", "Iq", "0jC", "64", "25D", "6Jb", "bEr", "46y", "539", "jm", "9Pf", "0gs", "I", "aCl", "bfn", +"bio", "72V", "1m2", "0hr", "325", "el", "5x0", "49x", "44H", "6Ec", "3nl", "95", "0eB", "Fp", "6kO", "5NM", "4mt", "5h4", "uh", "0VZ", +"0xv", "2MU", "4V9", "4CX", "4Nh", "7kj", "Vt", "01N", "m9", "xX", "6Ug", "54m", "4oE", "6Zf", "3Q", "b8", "0zG", "Yu", "60B", "4Ai", +"4LY", "4Y8", "TE", "0ww", "1Iz", "zi", "5g5", "4bu", "5y7", "5lV", "0GY", "dk", "JG", "0iu", "5Bz", "6gx", "6jH", "5OJ", "0dE", "Gw", +"3ok", "82", "45O", "6Dd", "6Ke", "5ng", "73", "fZ", "Hv", "0kD", "4Pj", "6eI", "6hy", "7m9", "0ft", "EF", "kj", "0HX", "4sv", "5v6", +"Wn", "00T", "4Or", "5J2", "407", "55w", "0Zp", "yB", "0z", "1Ga", "4ln", "6YM", "63i", "4BB", "0yl", "2LO", "2CN", "02e", "4MC", "7hA", +"6VL", "4co", "0XA", "2mb", "2K", "0Uq", "bTl", "a5f", "5E3", "5PR", "86M", "Xo", "11v", "Fm", "6kR", "5NP", "44U", "aol", "hA", "0Ks", +"0FC", "eq", "6HN", "49e", "4SA", "6fb", "3Mm", "0ho", "0gn", "T", "6ic", "5La", "46d", "6GO", "jp", "0IB", "0Dr", "1A2", "hyT", "bEo", +"4Qp", "5T0", "Il", "8cF", "0xr", "2MQ", "62w", "777", "4mp", "5h0", "1d", "9Og", "1KO", "2nM", "6Uc", "54i", "4Nl", "7kn", "Vp", "01J", +"0zC", "Yq", "60F", "4Am", "4oA", "6Zb", "3U", "0To", "8PG", "zm", "5g1", "4bq", "786", "aSl", "TA", "0ws", "JC", "0iq", "bhl", "73U", +"5y3", "5lR", "336", "do", "3oo", "86", "45K", "7TA", "6jL", "5ON", "0dA", "Gs", "Hr", "8bX", "4Pn", "6eM", "6Ka", "5nc", "77", "24G", +"kn", "8AD", "47z", "5v2", "aBo", "bgm", "0fp", "EB", "403", "4aZ", "0Zt", "yF", "Wj", "00P", "4Ov", "5J6", "63m", "4BF", "0yh", "ZZ", +"tv", "0WD", "4lj", "6YI", "6VH", "4ck", "0XE", "2mf", "2CJ", "02a", "4MG", "6xd", "5E7", "5PV", "1kx", "Xk", "2O", "0Uu", "bTh", "7KY", +"44Q", "4e8", "hE", "0Kw", "11r", "Fi", "6kV", "5NT", "4SE", "6ff", "KY", "0hk", "0FG", "eu", "6HJ", "49a", "4rh", "6GK", "jt", "0IF", +"Q9", "P", "6ig", "5Le", "4Qt", "5T4", "Ih", "0jZ", "0Dv", "gD", "4j9", "5oy", "aD0", "7kb", "3PL", "01F", "m1", "xP", "6Uo", "54e", +"59U", "a6E", "1h", "0VR", "85n", "196", "4V1", "4CP", "4LQ", "4Y0", "TM", "03w", "0YS", "za", "a9D", "56T", "4oM", "6Zn", "3Y", "b0", +"0zO", "2Ol", "60J", "4Aa", "7za", "5OB", "0dM", "2Qn", "iS", "0Ja", "45G", "6Dl", "acN", "48w", "0GQ", "dc", "JO", "94L", "4RS", "4G2", +"4H3", "5Ms", "12U", "EN", "kb", "0HP", "47v", "526", "6Km", "5no", "s3", "fR", "3NN", "0kL", "4Pb", "6eA", "0r", "0WH", "4lf", "6YE", +"63a", "4BJ", "O7", "ZV", "Wf", "0tT", "4Oz", "6zY", "4t7", "4aV", "0Zx", "yJ", "2C", "0Uy", "4nW", "7KU", "61P", "5PZ", "1kt", "Xg", +"UW", "02m", "4MK", "6xh", "6VD", "4cg", "0XI", "2mj", "0FK", "ey", "6HF", "49m", "4SI", "6fj", "KU", "0hg", "0eW", "Fe", "6kZ", "5NX", +"4pU", "4e4", "hI", "2k9", "0Dz", "gH", "4j5", "5ou", "4Qx", "5T8", "Id", "0jV", "Q5", "DT", "6ik", "5Li", "46l", "6GG", "jx", "0IJ", +"m5", "xT", "6Uk", "54a", "4Nd", "7kf", "Vx", "01B", "0xz", "192", "4V5", "4CT", "4mx", "5h8", "1l", "0VV", "0YW", "ze", "5g9", "4by", +"4LU", "4Y4", "TI", "03s", "0zK", "Yy", "60N", "4Ae", "4oI", "6Zj", "wU", "b4", "iW", "0Je", "45C", "6Dh", "6jD", "5OF", "0dI", "2Qj", +"JK", "0iy", "4RW", "4G6", "6IX", "48s", "0GU", "dg", "kf", "0HT", "47r", "522", "4H7", "5Mw", "0fx", "EJ", "Hz", "0kH", "4Pf", "6eE", +"6Ki", "5nk", "s7", "fV", "63e", "4BN", "O3", "ZR", "0v", "0WL", "4lb", "6YA", "4t3", "4aR", "8Sd", "yN", "Wb", "00X", "b1E", "aPO", +"61T", "bzL", "1kp", "Xc", "2G", "217", "4nS", "7KQ", "7Fa", "4cc", "0XM", "2mn", "US", "02i", "4MO", "6xl", "4SM", "6fn", "KQ", "0hc", +"0FO", "27d", "6HB", "49i", "44Y", "4e0", "hM", "8Bg", "0eS", "Fa", "aAL", "bdN", "656", "70v", "3OP", "0jR", "8Mf", "gL", "4j1", "5oq", +"46h", "6GC", "28e", "0IN", "Q1", "X", "6io", "5Lm", "6KV", "5nT", "1Uz", "fi", "HE", "0kw", "4PY", "4E8", "6hJ", "5MH", "0fG", "Eu", +"kY", "0Hk", "47M", "6Ff", "6Ig", "48L", "51", "dX", "Jt", "0iF", "4Rh", "6gK", "4J9", "5Oy", "0dv", "GD", "ih", "0JZ", "4qt", "5t4", +"4ov", "5j6", "3b", "0TX", "0zt", "YF", "60q", "4AZ", "4Lj", "6yI", "Tv", "03L", "0Yh", "zZ", "6We", "4bF", "4mG", "6Xd", "1S", "0Vi", +"0xE", "2Mf", "6vH", "4Ck", "bth", "7kY", "VG", "0uu", "1Kx", "xk", "5e7", "5pV", "13t", "g", "5Y3", "5LR", "46W", "amn", "jC", "0Iq", +"0DA", "gs", "6JL", "5oN", "4QC", "70I", "3Oo", "0jm", "0el", "2PO", "6ka", "5Nc", "44f", "6EM", "hr", "8BX", "0Fp", "eB", "abo", "49V", +"4Sr", "5V2", "Kn", "8aD", "Ul", "02V", "4Mp", "5H0", "425", "57u", "0Xr", "2mQ", "2x", "0UB", "4nl", "7Kn", "61k", "5Pa", "1kO", "2NM", +"2AL", "00g", "4OA", "6zb", "6TN", "4am", "0ZC", "yq", "0I", "0Ws", "58t", "a7d", "5G1", "4Bq", "84O", "Zm", "HA", "0ks", "bjn", "71W", +"6KR", "5nP", "314", "fm", "29D", "0Ho", "47I", "6Fb", "6hN", "5ML", "0fC", "Eq", "Jp", "0iB", "4Rl", "6gO", "6Ic", "48H", "55", "26E", +"il", "8CF", "45x", "508", "hYT", "beo", "0dr", "1a2", "0zp", "YB", "60u", "755", "4or", "5j2", "3f", "9Me", "0Yl", "2lO", "6Wa", "4bB", +"4Ln", "6yM", "Tr", "03H", "0xA", "2Mb", "62D", "4Co", "4mC", "7HA", "1W", "0Vm", "8RE", "xo", "5e3", "54Z", "b0d", "aQn", "VC", "01y", +"46S", "6Gx", "jG", "0Iu", "0gY", "c", "5Y7", "5LV", "4QG", "6dd", "3Ok", "0ji", "0DE", "gw", "6JH", "5oJ", "44b", "6EI", "hv", "0KD", +"0eh", "FZ", "6ke", "5Ng", "4Sv", "5V6", "Kj", "0hX", "0Ft", "eF", "6Hy", "49R", "421", "4cX", "0Xv", "2mU", "Uh", "02R", "4Mt", "5H4", +"61o", "5Pe", "M9", "XX", "vt", "0UF", "4nh", "7Kj", "6TJ", "4ai", "0ZG", "yu", "WY", "B8", "4OE", "6zf", "5G5", "4Bu", "1iz", "Zi", +"0M", "0Ww", "4lY", "4y8", "6hB", "aW1", "0fO", "2Sl", "kQ", "0Hc", "47E", "6Fn", "aaL", "bDN", "0ES", "fa", "HM", "8bg", "4PQ", "4E0", +"4J1", "5Oq", "10W", "GL", "3oP", "0JR", "45t", "504", "6Io", "48D", "59", "dP", "3LL", "0iN", "5BA", "6gC", "4Lb", "6yA", "2Bo", "03D", +"o3", "zR", "6Wm", "4bN", "bUM", "a4G", "3j", "0TP", "87l", "YN", "4T3", "4AR", "4NS", "7kQ", "VO", "01u", "1Kp", "xc", "hfw", "54V", +"4mO", "6Xl", "uS", "0Va", "0xM", "2Mn", "62H", "4Cc", "0DI", "25b", "6JD", "5oF", "4QK", "6dh", "IW", "0je", "0gU", "o", "6iX", "5LZ", +"4rW", "4g6", "jK", "0Iy", "0Fx", "eJ", "4h7", "5mw", "4Sz", "6fY", "Kf", "0hT", "S7", "FV", "6ki", "5Nk", "44n", "6EE", "hz", "0KH", +"2p", "0UJ", "4nd", "7Kf", "61c", "5Pi", "M5", "XT", "Ud", "0vV", "4Mx", "5H8", "4v5", "4cT", "0Xz", "2mY", "0A", "1GZ", "4lU", "4y4", +"5G9", "4By", "0yW", "Ze", "WU", "B4", "4OI", "6zj", "6TF", "4ae", "0ZK", "yy", "kU", "0Hg", "47A", "6Fj", "6hF", "5MD", "0fK", "Ey", +"HI", "2K9", "4PU", "4E4", "6KZ", "5nX", "0EW", "fe", "id", "0JV", "45p", "500", "4J5", "5Ou", "0dz", "GH", "Jx", "0iJ", "4Rd", "6gG", +"6Ik", "5li", "q5", "dT", "o7", "zV", "6Wi", "4bJ", "4Lf", "6yE", "Tz", "0wH", "0zx", "YJ", "4T7", "4AV", "4oz", "6ZY", "3n", "0TT", +"1Kt", "xg", "6UX", "54R", "4NW", "7kU", "VK", "01q", "0xI", "2Mj", "62L", "4Cg", "4mK", "6Xh", "uW", "0Ve", "4QO", "6dl", "IS", "0ja", +"0DM", "25f", "7Za", "5oB", "4rS", "4g2", "jO", "9PD", "0gQ", "k", "aCN", "685", "674", "72t", "Kb", "0hP", "8Od", "eN", "4h3", "49Z", +"44j", "6EA", "3nN", "0KL", "S3", "FR", "6km", "5No", "61g", "5Pm", "M1", "XP", "2t", "0UN", "ad0", "7Kb", "429", "4cP", "8Qf", "39t", +"0c3", "02Z", "b3G", "aRM", "63V", "bxN", "0yS", "Za", "0E", "235", "4lQ", "4y0", "6TB", "4aa", "0ZO", "2ol", "WQ", "B0", "4OM", "6zn", +"4i4", "5lt", "1WZ", "dI", "Je", "0iW", "4Ry", "5W9", "6jj", "5Oh", "R4", "GU", "iy", "0JK", "45m", "6DF", "6KG", "5nE", "0EJ", "fx", +"HT", "0kf", "4PH", "6ek", "5X8", "5MY", "0fV", "Ed", "kH", "0Hz", "4sT", "4f5", "4mV", "4x7", "1B", "0Vx", "0xT", "0m5", "62Q", "4Cz", +"4NJ", "7kH", "VV", "C7", "1Ki", "xz", "6UE", "54O", "4og", "6ZD", "3s", "0TI", "L6", "YW", "6th", "4AK", "6l9", "6yX", "Tg", "0wU", +"0Yy", "zK", "4w6", "4bW", "11T", "FO", "4K2", "5Nr", "44w", "517", "hc", "0KQ", "p2", "eS", "6Hl", "49G", "4Sc", "72i", "3MO", "0hM", +"0gL", "v", "6iA", "5LC", "46F", "6Gm", "jR", "1YA", "0DP", "gb", "hyv", "bEM", "4QR", "4D3", "IN", "8cd", "WL", "00v", "4OP", "4Z1", +"hgt", "55U", "0ZR", "0O3", "0X", "a1", "4lL", "6Yo", "63K", "5RA", "0yN", "2Lm", "2Cl", "02G", "4Ma", "6xB", "6Vn", "4cM", "n0", "39i", +"2i", "0US", "bTN", "a5D", "4U0", "5Pp", "86o", "XM", "Ja", "0iS", "667", "73w", "4i0", "48Y", "8Ng", "dM", "3oM", "0JO", "45i", "6DB", +"6jn", "5Ol", "R0", "GQ", "HP", "0kb", "4PL", "6eo", "6KC", "5nA", "0EN", "24e", "kL", "8Af", "47X", "4f1", "aBM", "696", "0fR", "0s3", +"0xP", "0m1", "62U", "byM", "4mR", "4x3", "1F", "226", "1Km", "2no", "6UA", "54K", "4NN", "7kL", "VR", "C3", "L2", "YS", "60d", "4AO", +"4oc", "7Ja", "3w", "0TM", "8Pe", "zO", "4w2", "4bS", "b2D", "aSN", "Tc", "03Y", "44s", "513", "hg", "0KU", "0ey", "FK", "4K6", "5Nv", +"4Sg", "6fD", "3MK", "0hI", "p6", "eW", "6Hh", "49C", "46B", "6Gi", "jV", "0Id", "0gH", "r", "6iE", "5LG", "4QV", "4D7", "IJ", "0jx", +"0DT", "gf", "6JY", "bEI", "5d8", "4ax", "0ZV", "yd", "WH", "00r", "4OT", "4Z5", "63O", "4Bd", "0yJ", "Zx", "tT", "a5", "4lH", "6Yk", +"6Vj", "4cI", "n4", "2mD", "Uy", "02C", "4Me", "6xF", "4U4", "5Pt", "1kZ", "XI", "2m", "0UW", "4ny", "5k9", "6jb", "ber", "0do", "2QL", +"iq", "0JC", "45e", "6DN", "acl", "48U", "0Gs", "dA", "Jm", "94n", "4Rq", "5W1", "5X0", "5MQ", "12w", "El", "1M2", "0Hr", "47T", "alm", +"6KO", "5nM", "0EB", "fp", "3Nl", "0kn", "bjs", "6ec", "4NB", "aQs", "3Pn", "01d", "1Ka", "xr", "6UM", "54G", "59w", "a6g", "1J", "0Vp", +"85L", "d3D", "5F2", "4Cr", "4Ls", "5I3", "To", "03U", "0Yq", "zC", "436", "56v", "4oo", "6ZL", "ws", "0TA", "0zm", "2ON", "60h", "4AC", +"42", "27B", "6Hd", "49O", "4Sk", "6fH", "Kw", "0hE", "0eu", "FG", "6kx", "5Nz", "4pw", "5u7", "hk", "0KY", "0DX", "gj", "5z6", "5oW", +"4QZ", "6dy", "IF", "0jt", "0gD", "Dv", "6iI", "5LK", "46N", "6Ge", "jZ", "0Ih", "0P", "a9", "4lD", "6Yg", "63C", "4Bh", "0yF", "Zt", +"WD", "0tv", "4OX", "4Z9", "5d4", "4at", "0ZZ", "yh", "2a", "1Ez", "4nu", "5k5", "4U8", "5Px", "1kV", "XE", "Uu", "02O", "4Mi", "6xJ", +"6Vf", "4cE", "n8", "2mH", "iu", "0JG", "45a", "6DJ", "6jf", "5Od", "R8", "GY", "Ji", "1yz", "4Ru", "5W5", "4i8", "48Q", "0Gw", "dE", +"kD", "0Hv", "47P", "4f9", "5X4", "5MU", "0fZ", "Eh", "HX", "0kj", "4PD", "6eg", "6KK", "5nI", "0EF", "ft", "1Ke", "xv", "6UI", "54C", +"4NF", "7kD", "VZ", "0uh", "0xX", "0m9", "5F6", "4Cv", "4mZ", "6Xy", "1N", "0Vt", "0Yu", "zG", "432", "56r", "4Lw", "5I7", "Tk", "03Q", +"0zi", "2OJ", "60l", "4AG", "4ok", "6ZH", "ww", "0TE", "4So", "6fL", "Ks", "0hA", "46", "27F", "7XA", "49K", "4ps", "5u3", "ho", "8BE", +"0eq", "FC", "aAn", "bdl", "bkm", "70T", "IB", "0jp", "307", "gn", "5z2", "5oS", "46J", "6Ga", "28G", "0Il", "13i", "z", "6iM", "5LO", +"63G", "4Bl", "0yB", "Zp", "0T", "0Wn", "58i", "6Yc", "5d0", "4ap", "8SF", "yl", "1q2", "00z", "b1g", "aPm", "61v", "746", "1kR", "XA", +"2e", "9Lf", "4nq", "5k1", "6Vb", "4cA", "0Xo", "2mL", "Uq", "02K", "4Mm", "6xN", "8YG", "7e", "5n1", "4kq", "716", "64v", "2KP", "1nR", +"07K", "Pq", "69F", "4Hm", "4fA", "6Sb", "2hL", "1MN", "0Rn", "5T", "7LB", "5ya", "4Gl", "66G", "2Ia", "08J", "05z", "1t2", "aUm", "b4g", +"4dp", "5a0", "8d", "8VF", "bn", "357", "4zr", "6OQ", "75T", "bnm", "0op", "LB", "Ar", "16i", "4Yn", "6lM", "6Ba", "43J", "0Ll", "2yO", +"22F", "16", "4xC", "agr", "6cL", "4Vo", "0mA", "Ns", "CC", "14X", "bal", "aDn", "5p3", "4us", "8GE", "mo", "5L7", "4Iw", "06Q", "Qk", +"1Y5", "1LT", "53r", "462", "7Oi", "4jk", "0QE", "rw", "2JJ", "1oH", "4DG", "65l", "7nD", "4KF", "0ph", "SZ", "2kg", "1Ne", "4ej", "6PI", +"493", "4hZ", "0St", "4N", "0h9", "09P", "4Fv", "5C6", "4Xt", "6mW", "023", "0cZ", "0Mv", "nD", "4c9", "42P", "5kI", "6NK", "ct", "1Pg", +"X9", "MX", "74N", "4UD", "4ZE", "6of", "BY", "W8", "0OG", "lu", "6AJ", "40a", "4yY", "4l8", "aE", "0Bw", "18r", "Oi", "5R5", "4Wu", +"4EY", "4P8", "2KT", "1nV", "8YC", "7a", "5n5", "4ku", "4fE", "6Sf", "2hH", "k8", "07O", "Pu", "69B", "4Hi", "4Gh", "66C", "2Ie", "08N", +"d9", "5P", "7LF", "4iD", "4dt", "5a4", "2jy", "3o9", "0qv", "RD", "7oZ", "4JX", "6ay", "4TZ", "0ot", "LF", "bj", "0AX", "4zv", "6OU", +"6Be", "43N", "0Lh", "oZ", "Av", "0bD", "4Yj", "6lI", "6cH", "4Vk", "0mE", "Nw", "22B", "12", "4xG", "6Md", "5p7", "4uw", "0NY", "mk", +"CG", "1pT", "5Kz", "6nx", "1Y1", "1LP", "53v", "466", "5L3", "4Is", "06U", "Qo", "2JN", "1oL", "4DC", "65h", "7Om", "4jo", "0QA", "rs", +"9z", "1Na", "4en", "6PM", "aTs", "4KB", "04d", "2EO", "d6D", "09T", "4Fr", "5C2", "497", "bRm", "0Sp", "4J", "0Mr", "1H2", "aim", "42T", +"4Xp", "6mS", "027", "17w", "0nn", "3Kl", "74J", "5Ea", "5kM", "6NO", "cp", "1Pc", "0OC", "lq", "6AN", "40e", "4ZA", "6ob", "2TL", "0ao", +"18v", "Om", "5R1", "4Wq", "bCn", "afl", "aA", "0Bs", "07C", "Py", "69N", "4He", "4fI", "6Sj", "2hD", "k4", "0PW", "7m", "5n9", "4ky", +"4EU", "4P4", "2KX", "1nZ", "05r", "RH", "7oV", "4JT", "4dx", "5a8", "8l", "1Ow", "d5", "qT", "7LJ", "4iH", "4Gd", "66O", "2Ii", "08B", +"Az", "0bH", "4Yf", "6lE", "6Bi", "43B", "z7", "oV", "bf", "0AT", "4zz", "6OY", "4A7", "4TV", "0ox", "LJ", "CK", "14P", "5Kv", "4N6", +"543", "41s", "0NU", "mg", "22N", "u6", "4xK", "6Mh", "6cD", "4Vg", "0mI", "2Xj", "7Oa", "4jc", "0QM", "6w", "2JB", "I2", "4DO", "65d", +"68T", "b7D", "06Y", "Qc", "dSm", "287", "4gS", "4r2", "7MP", "4hR", "276", "4F", "0h1", "09X", "b8E", "67U", "7nL", "4KN", "F3", "SR", +"9v", "1Nm", "4eb", "6PA", "5kA", "6NC", "21e", "1Po", "X1", "MP", "74F", "4UL", "bbO", "79v", "0v3", "0cR", "8Df", "nL", "4c1", "42X", +"4yQ", "4l0", "aM", "8Kg", "0lS", "Oa", "76w", "637", "4ZM", "6on", "BQ", "W0", "0OO", "2zl", "6AB", "40i", "4fM", "6Sn", "3xa", "k0", +"07G", "2Fl", "69J", "4Ha", "4EQ", "4P0", "d5g", "83o", "0PS", "7i", "a0D", "bQN", "50U", "hbt", "8h", "1Os", "05v", "RL", "7oR", "4JP", +"5WA", "66K", "2Im", "08F", "d1", "5X", "7LN", "4iL", "6Bm", "43F", "z3", "oR", "2Wo", "0bL", "4Yb", "6lA", "4A3", "4TR", "8fd", "LN", +"bb", "0AP", "cPl", "aeO", "547", "41w", "0NQ", "mc", "CO", "14T", "5Kr", "4N2", "77i", "4Vc", "0mM", "2Xn", "22J", "u2", "4xO", "6Ml", +"2JF", "I6", "4DK", "6qh", "7Oe", "4jg", "0QI", "6s", "1Y9", "1LX", "4gW", "4r6", "68P", "5YZ", "0rU", "Qg", "0h5", "1mu", "4Fz", "67Q", +"7MT", "4hV", "0Sx", "4B", "9r", "1Ni", "4ef", "6PE", "7nH", "4KJ", "F7", "SV", "X5", "MT", "74B", "4UH", "5kE", "6NG", "cx", "1Pk", +"0Mz", "nH", "4c5", "4vT", "4Xx", "79r", "0v7", "0cV", "0lW", "Oe", "5R9", "4Wy", "4yU", "4l4", "aI", "1RZ", "0OK", "ly", "6AF", "40m", +"4ZI", "6oj", "BU", "W4", "265", "5E", "488", "4iQ", "b9F", "66V", "0i2", "1lr", "G0", "RQ", "7oO", "4JM", "4da", "6QB", "8u", "1On", +"0PN", "7t", "7Nb", "aa0", "4EL", "64g", "2KA", "H1", "07Z", "0f3", "69W", "b6G", "4fP", "479", "dRn", "294", "22W", "8Jd", "4xR", "4m3", +"77t", "624", "0mP", "Nb", "CR", "V3", "5Ko", "6nm", "ajS", "41j", "0NL", "3kN", "20f", "0AM", "4zc", "aeR", "6al", "4TO", "Y2", "LS", +"Ac", "0bQ", "bcL", "78u", "4b2", "4wS", "8Ee", "oO", "7nU", "4KW", "04q", "SK", "9o", "1Nt", "51R", "6PX", "7MI", "4hK", "e6", "pW", +"2Hj", "09A", "4Fg", "67L", "68M", "4If", "0rH", "Qz", "2iG", "j7", "4gJ", "6Ri", "7Ox", "4jz", "0QT", "6n", "1z8", "1oY", "4DV", "4Q7", +"4ZT", "4O5", "BH", "0az", "0OV", "ld", "550", "40p", "4yH", "6Lk", "aT", "t5", "0lJ", "Ox", "6bG", "4Wd", "4Xe", "6mF", "2Vh", "0cK", +"0Mg", "nU", "6Cj", "42A", "5kX", "6NZ", "ce", "1Pv", "2N9", "MI", "7pW", "4UU", "4Gy", "5B9", "0i6", "1lv", "1BZ", "5A", "7LW", "4iU", +"4de", "6QF", "8q", "1Oj", "G4", "RU", "7oK", "4JI", "4EH", "64c", "2KE", "H5", "0PJ", "7p", "7Nf", "4kd", "4fT", "4s5", "2hY", "290", +"0sV", "Pd", "5M8", "4Hx", "6cY", "4Vz", "0mT", "Nf", "1F8", "0Cx", "4xV", "4m7", "7Pd", "41n", "0NH", "mz", "CV", "V7", "5Kk", "6ni", +"6ah", "4TK", "Y6", "LW", "20b", "0AI", "4zg", "6OD", "4b6", "4wW", "0Ly", "oK", "Ag", "0bU", "5IZ", "6lX", "9k", "1Np", "51V", "azN", +"7nQ", "4KS", "04u", "SO", "2Hn", "09E", "4Fc", "67H", "7MM", "4hO", "e2", "pS", "2iC", "j3", "4gN", "6Rm", "68I", "4Ib", "06D", "2Go", +"d4d", "82l", "4DR", "4Q3", "a1G", "bPM", "0QP", "6j", "0OR", "0Z3", "554", "40t", "4ZP", "4O1", "BL", "15W", "0lN", "2Ym", "6bC", "5GA", +"4yL", "6Lo", "aP", "09", "0Mc", "nQ", "6Cn", "42E", "4Xa", "6mB", "2Vl", "0cO", "8gg", "MM", "7pS", "4UQ", "bAN", "adL", "ca", "1Pr", +"G8", "RY", "7oG", "4JE", "4di", "6QJ", "2jd", "1Of", "0Rw", "5M", "480", "4iY", "4Gu", "5B5", "2Ix", "08S", "07R", "Ph", "5M4", "4Ht", +"4fX", "471", "1X6", "1MW", "0PF", "st", "7Nj", "4kh", "4ED", "64o", "2KI", "H9", "CZ", "14A", "5Kg", "6ne", "7Ph", "41b", "0ND", "mv", +"1F4", "0Ct", "4xZ", "6My", "5S6", "4Vv", "0mX", "Nj", "Ak", "0bY", "4Yw", "6lT", "6Bx", "43S", "0Lu", "oG", "bw", "0AE", "4zk", "6OH", +"6ad", "4TG", "0oi", "2ZJ", "7MA", "4hC", "0Sm", "4W", "2Hb", "09I", "4Fo", "67D", "aTn", "b5d", "04y", "SC", "9g", "8WE", "4es", "6PP", +"5o2", "4jr", "8XD", "6f", "1z0", "1oQ", "705", "65u", "68E", "4In", "06H", "Qr", "2iO", "1LM", "4gB", "6Ra", "5ia", "6Lc", "23E", "05", +"0lB", "Op", "6bO", "4Wl", "c4F", "aEm", "1d2", "0ar", "8FF", "ll", "558", "40x", "5kP", "6NR", "cm", "344", "0ns", "MA", "74W", "bon", +"4Xm", "6mN", "3FA", "0cC", "0Mo", "2xL", "6Cb", "42I", "4dm", "6QN", "8y", "1Ob", "05g", "2DL", "7oC", "4JA", "4Gq", "5B1", "d7G", "08W", +"0Rs", "5I", "484", "bSn", "52u", "475", "1X2", "1MS", "07V", "Pl", "5M0", "4Hp", "5Ua", "64k", "2KM", "1nO", "0PB", "7x", "7Nn", "4kl", +"7Pl", "41f", "8GX", "mr", "2UO", "14E", "5Kc", "6na", "5S2", "4Vr", "19u", "Nn", "1F0", "0Cp", "bBm", "ago", "ahn", "43W", "0Lq", "oC", +"Ao", "16t", "4Ys", "6lP", "75I", "4TC", "0om", "2ZN", "bs", "0AA", "4zo", "6OL", "2Hf", "09M", "4Fk", "6sH", "7ME", "4hG", "0Si", "4S", +"9c", "1Nx", "4ew", "6PT", "7nY", "bqh", "0pu", "SG", "1z4", "1oU", "4DZ", "65q", "5o6", "4jv", "0QX", "6b", "2iK", "1LI", "4gF", "6Re", +"68A", "4Ij", "06L", "Qv", "0lF", "Ot", "6bK", "4Wh", "4yD", "6Lg", "aX", "01", "0OZ", "lh", "5q4", "4tt", "4ZX", "4O9", "BD", "0av", +"0nw", "ME", "74S", "4UY", "5kT", "6NV", "ci", "1Pz", "0Mk", "nY", "6Cf", "42M", "4Xi", "6mJ", "2Vd", "0cG", "bL", "8Hf", "4zP", "4o1", +"75v", "606", "0oR", "0z3", "AP", "T1", "4YL", "6lo", "6BC", "43h", "0LN", "2ym", "22d", "0CO", "4xa", "6MB", "6cn", "4VM", "0mc", "NQ", +"Ca", "14z", "baN", "aDL", "7PS", "41Y", "8Gg", "mM", "247", "7G", "7NQ", "4kS", "com", "64T", "0k0", "1np", "E2", "PS", "69d", "4HO", +"4fc", "7Ca", "2hn", "1Ml", "0RL", "5v", "avS", "4ib", "4GN", "66e", "2IC", "J3", "05X", "Rb", "aUO", "b4E", "4dR", "4q3", "8F", "8Vd", +"4XV", "4M7", "1f8", "0cx", "0MT", "nf", "572", "42r", "5kk", "6Ni", "cV", "v7", "0nH", "Mz", "74l", "4Uf", "4Zg", "6oD", "2Tj", "0aI", +"y6", "lW", "6Ah", "40C", "5iZ", "583", "ag", "0BU", "0ly", "OK", "4B6", "4WW", "7lW", "4IU", "06s", "QI", "0I6", "1Lv", "4gy", "5b9", +"7OK", "4jI", "g4", "rU", "2Jh", "1oj", "4De", "65N", "7nf", "4Kd", "04B", "Sx", "2kE", "h5", "4eH", "6Pk", "5m8", "4hx", "0SV", "4l", +"2HY", "09r", "4FT", "4S5", "5Q8", "4Tx", "0oV", "Ld", "bH", "0Az", "4zT", "4o5", "6BG", "43l", "0LJ", "ox", "AT", "T5", "4YH", "6lk", +"6cj", "4VI", "0mg", "NU", "2vh", "0CK", "4xe", "6MF", "7PW", "4uU", "2n9", "mI", "Ce", "1pv", "5KX", "6nZ", "5UZ", "64P", "0k4", "1nt", +"0Py", "7C", "7NU", "4kW", "4fg", "6SD", "2hj", "1Mh", "E6", "PW", "7mI", "4HK", "4GJ", "66a", "2IG", "J7", "0RH", "5r", "7Ld", "4if", +"4dV", "4q7", "8B", "1OY", "0qT", "Rf", "7ox", "4Jz", "0MP", "nb", "576", "42v", "4XR", "4M3", "dll", "17U", "0nL", "3KN", "74h", "4Ub", +"5ko", "6Nm", "cR", "v3", "y2", "lS", "6Al", "40G", "4Zc", "aER", "2Tn", "0aM", "18T", "OO", "4B2", "4WS", "bCL", "587", "ac", "0BQ", +"0I2", "1Lr", "53T", "axL", "68z", "4IQ", "06w", "QM", "2Jl", "1on", "4Da", "65J", "7OO", "4jM", "g0", "6Y", "9X", "h1", "4eL", "6Po", +"7nb", "aA0", "04F", "2Em", "d6f", "09v", "4FP", "4S1", "a3E", "bRO", "0SR", "4h", "AX", "T9", "4YD", "6lg", "6BK", "4wh", "0LF", "ot", +"bD", "0Av", "4zX", "4o9", "5Q4", "4Tt", "0oZ", "Lh", "Ci", "14r", "5KT", "6nV", "ajh", "41Q", "0Nw", "mE", "22l", "0CG", "4xi", "6MJ", +"6cf", "4VE", "0mk", "NY", "07a", "2FJ", "69l", "4HG", "4fk", "6SH", "2hf", "1Md", "0Pu", "7O", "7NY", "bQh", "4Ew", "6pT", "0k8", "1nx", +"05P", "Rj", "5O6", "4Jv", "4dZ", "453", "8N", "1OU", "0RD", "qv", "7Lh", "4ij", "4GF", "66m", "2IK", "1lI", "5kc", "6Na", "21G", "27", +"8gX", "Mr", "74d", "4Un", "bbm", "79T", "1f0", "0cp", "397", "nn", "5s2", "42z", "4ys", "6LP", "ao", "366", "0lq", "OC", "76U", "bml", +"4Zo", "6oL", "Bs", "0aA", "0Om", "2zN", "7QA", "40K", "7OC", "4jA", "0Qo", "6U", "3ZA", "1ob", "4Dm", "65F", "68v", "b7f", "0rs", "QA", +"dSO", "8UG", "4gq", "5b1", "5m0", "4hp", "8ZF", "4d", "1x2", "09z", "727", "67w", "7nn", "4Kl", "04J", "Sp", "9T", "1NO", "51i", "6Pc", +"6BO", "43d", "0LB", "op", "2WM", "0bn", "5Ia", "6lc", "5Q0", "4Tp", "8fF", "Ll", "1D2", "0Ar", "cPN", "aem", "ajl", "41U", "0Ns", "mA", +"Cm", "14v", "5KP", "6nR", "6cb", "4VA", "0mo", "2XL", "22h", "0CC", "4xm", "6MN", "4fo", "6SL", "2hb", "8TY", "07e", "2FN", "69h", "4HC", +"4Es", "64X", "d5E", "83M", "0Pq", "7K", "a0f", "bQl", "50w", "457", "8J", "1OQ", "05T", "Rn", "5O2", "4Jr", "4GB", "66i", "2IO", "08d", +"1Ba", "5z", "7Ll", "4in", "0nD", "Mv", "7ph", "4Uj", "5kg", "6Ne", "cZ", "23", "0MX", "nj", "5s6", "4vv", "4XZ", "6my", "1f4", "0ct", +"0lu", "OG", "6bx", "5Gz", "4yw", "6LT", "ak", "0BY", "0Oi", "2zJ", "6Ad", "40O", "4Zk", "6oH", "Bw", "0aE", "2Jd", "1of", "4Di", "65B", +"7OG", "4jE", "g8", "6Q", "2ix", "1Lz", "4gu", "5b5", "68r", "4IY", "0rw", "QE", "1x6", "1mW", "4FX", "4S9", "5m4", "4ht", "0SZ", "ph", +"9P", "h9", "4eD", "6Pg", "7nj", "4Kh", "04N", "St", "22u", "375", "4xp", "598", "77V", "blo", "0mr", "1h2", "Cp", "14k", "5KM", "6nO", +"7PB", "41H", "0Nn", "3kl", "20D", "34", "4zA", "6Ob", "6aN", "4Tm", "0oC", "Lq", "AA", "0bs", "bcn", "78W", "569", "43y", "384", "om", +"9Kd", "5g", "5l3", "4is", "734", "66t", "1y1", "08y", "05I", "Rs", "7om", "4Jo", "4dC", "7AA", "8W", "1OL", "0Pl", "7V", "ats", "4kB", +"4En", "64E", "2Kc", "1na", "07x", "PB", "69u", "b6e", "4fr", "5c2", "dRL", "8TD", "4Zv", "6oU", "Bj", "0aX", "0Ot", "lF", "6Ay", "40R", +"4yj", "6LI", "av", "0BD", "0lh", "OZ", "6be", "4WF", "4XG", "6md", "2VJ", "0ci", "0ME", "nw", "6CH", "42c", "5kz", "6Nx", "cG", "1PT", +"0nY", "Mk", "5P7", "4Uw", "5N5", "4Ku", "04S", "Si", "9M", "1NV", "4eY", "440", "7Mk", "4hi", "0SG", "pu", "2HH", "K8", "4FE", "67n", +"68o", "4ID", "1", "QX", "2ie", "1Lg", "4gh", "6RK", "7OZ", "4jX", "0Qv", "6L", "2Jy", "3O9", "4Dt", "5A4", "4C9", "4VX", "0mv", "ND", +"22q", "0CZ", "4xt", "6MW", "7PF", "41L", "x9", "mX", "Ct", "14o", "5KI", "6nK", "6aJ", "4Ti", "0oG", "Lu", "bY", "30", "4zE", "6Of", +"5r5", "4wu", "380", "oi", "AE", "0bw", "4YY", "4L8", "5Wz", "66p", "1y5", "1lT", "0RY", "5c", "5l7", "4iw", "4dG", "6Qd", "8S", "1OH", +"05M", "Rw", "7oi", "4Jk", "4Ej", "64A", "2Kg", "1ne", "0Ph", "7R", "7ND", "4kF", "4fv", "5c6", "0H9", "1My", "0st", "PF", "69q", "4HZ", +"0Op", "lB", "ako", "40V", "4Zr", "6oQ", "Bn", "15u", "0ll", "2YO", "6ba", "4WB", "4yn", "6LM", "ar", "1Ra", "0MA", "ns", "6CL", "42g", +"4XC", "79I", "2VN", "0cm", "8gE", "Mo", "5P3", "4Us", "bAl", "adn", "cC", "1PP", "9I", "1NR", "51t", "444", "5N1", "4Kq", "04W", "Sm", +"2HL", "09g", "4FA", "67j", "7Mo", "4hm", "0SC", "4y", "2ia", "1Lc", "4gl", "6RO", "68k", "5Ya", "5", "2GM", "d4F", "82N", "4Dp", "5A0", +"a1e", "bPo", "0Qr", "6H", "Cx", "14c", "5KE", "6nG", "7PJ", "4uH", "x5", "mT", "0V7", "0CV", "4xx", "590", "4C5", "4VT", "0mz", "NH", +"AI", "16R", "4YU", "4L4", "561", "43q", "0LW", "oe", "bU", "w4", "4zI", "6Oj", "6aF", "4Te", "0oK", "Ly", "05A", "2Dj", "7oe", "4Jg", +"4dK", "6Qh", "2jF", "i6", "0RU", "5o", "7Ly", "5yZ", "4GW", "4R6", "1y9", "08q", "07p", "PJ", "7mT", "4HV", "4fz", "6SY", "0H5", "1Mu", +"f7", "sV", "7NH", "4kJ", "4Ef", "64M", "2Kk", "1ni", "4yb", "6LA", "23g", "0BL", "Z3", "OR", "6bm", "4WN", "c4d", "aEO", "Bb", "0aP", +"8Fd", "lN", "4a3", "40Z", "5kr", "4n2", "cO", "8Ie", "0nQ", "Mc", "74u", "615", "4XO", "6ml", "2VB", "U2", "0MM", "2xn", "7Sa", "42k", +"7Mc", "4ha", "0SO", "4u", "3Xa", "K0", "4FM", "67f", "aTL", "b5F", "0pS", "Sa", "9E", "8Wg", "4eQ", "448", "7OR", "4jP", "254", "6D", +"0j3", "1os", "cnn", "65W", "68g", "4IL", "9", "QP", "2im", "1Lo", "53I", "6RC", "7PN", "41D", "x1", "mP", "2Um", "14g", "5KA", "6nC", +"4C1", "4VP", "19W", "NL", "0V3", "0CR", "bBO", "594", "565", "43u", "0LS", "oa", "AM", "16V", "4YQ", "4L0", "6aB", "4Ta", "0oO", "2Zl", +"bQ", "38", "4zM", "6On", "4dO", "6Ql", "2jB", "i2", "05E", "2Dn", "7oa", "4Jc", "4GS", "4R2", "d7e", "08u", "0RQ", "5k", "a2F", "bSL", +"52W", "ayO", "0H1", "1Mq", "07t", "PN", "69y", "4HR", "4Eb", "64I", "2Ko", "1nm", "f3", "7Z", "7NL", "4kN", "Z7", "OV", "6bi", "4WJ", +"4yf", "6LE", "az", "0BH", "0Ox", "lJ", "4a7", "4tV", "4Zz", "6oY", "Bf", "0aT", "0nU", "Mg", "74q", "5EZ", "5kv", "4n6", "cK", "1PX", +"0MI", "2xj", "6CD", "42o", "4XK", "6mh", "2VF", "U6", "2HD", "K4", "4FI", "67b", "7Mg", "4he", "0SK", "4q", "9A", "1NZ", "4eU", "4p4", +"5N9", "4Ky", "0pW", "Se", "0j7", "1ow", "4Dx", "5A8", "7OV", "4jT", "0Qz", "rH", "2ii", "1Lk", "4gd", "6RG", "68c", "4IH", "D5", "QT", +"5Ls", "4I3", "F", "13U", "0IP", "jb", "536", "46v", "5oo", "6Jm", "gR", "r3", "0jL", "3ON", "6dA", "4Qb", "5NB", "aAR", "2Pn", "0eM", +"0Ka", "hS", "6El", "44G", "49w", "abN", "ec", "0FQ", "8ae", "KO", "4F2", "4SS", "4X0", "4MQ", "02w", "UM", "0M2", "0XS", "57T", "a8D", +"7KO", "4nM", "c0", "2Y", "2Nl", "1kn", "aJ1", "61J", "6zC", "aE0", "00F", "2Am", "yP", "l1", "4aL", "6To", "a7E", "58U", "0WR", "0h", +"ZL", "84n", "4BP", "4W1", "fH", "0Ez", "5nu", "4k5", "5U8", "4Px", "0kV", "Hd", "ET", "P5", "5Mi", "6hk", "6FG", "47l", "0HJ", "kx", +"dy", "0GK", "48m", "6IF", "6gj", "4RI", "0ig", "JU", "Ge", "0dW", "5OX", "5Z9", "4d4", "4qU", "1ZZ", "iI", "0Ty", "3C", "4z6", "4oW", +"5QZ", "60P", "Yg", "0zU", "A6", "TW", "6yh", "4LK", "4bg", "6WD", "2lj", "0YI", "0VH", "1r", "6XE", "4mf", "4CJ", "62a", "2MG", "N7", +"0uT", "Vf", "7kx", "4Nz", "5pw", "4u7", "xJ", "1KY", "0IT", "jf", "532", "46r", "5Lw", "4I7", "B", "0gx", "0jH", "Iz", "6dE", "4Qf", +"5ok", "6Ji", "gV", "r7", "0Ke", "hW", "6Eh", "44C", "5NF", "6kD", "2Pj", "0eI", "0hy", "KK", "4F6", "4SW", "49s", "6HX", "eg", "0FU", +"0M6", "0XW", "4cy", "5f9", "4X4", "4MU", "02s", "UI", "Xy", "1kj", "5PD", "61N", "7KK", "4nI", "c4", "vU", "yT", "l5", "4aH", "6Tk", +"6zG", "4Od", "00B", "Wx", "ZH", "0yz", "4BT", "4W5", "5i8", "4lx", "0WV", "0l", "71v", "646", "0kR", "3NP", "fL", "8Lf", "5nq", "4k1", +"6FC", "47h", "0HN", "29e", "EP", "P1", "5Mm", "6ho", "6gn", "4RM", "0ic", "JQ", "26d", "0GO", "48i", "6IB", "4d0", "45Y", "8Cg", "iM", +"Ga", "0dS", "beN", "hYu", "ckm", "60T", "Yc", "0zQ", "207", "3G", "4z2", "4oS", "4bc", "7Ga", "2ln", "0YM", "A2", "TS", "6yl", "4LO", +"4CN", "62e", "2MC", "N3", "0VL", "1v", "6XA", "4mb", "5ps", "4u3", "xN", "8Rd", "01X", "Vb", "aQO", "b0E", "5og", "6Je", "gZ", "63", +"0jD", "Iv", "6dI", "4Qj", "7l9", "6iy", "N", "0gt", "0IX", "jj", "5w6", "4rv", "5mV", "5x7", "ek", "0FY", "0hu", "KG", "6fx", "5Cz", +"5NJ", "6kH", "Fw", "0eE", "92", "3nk", "6Ed", "44O", "7KG", "4nE", "c8", "2Q", "Xu", "1kf", "5PH", "61B", "4X8", "4MY", "0vw", "UE", +"2mx", "1Hz", "4cu", "5f5", "5i4", "4lt", "0WZ", "th", "ZD", "0yv", "4BX", "4W9", "6zK", "4Oh", "00N", "Wt", "yX", "l9", "4aD", "6Tg", +"2SM", "0fn", "5Ma", "6hc", "6FO", "47d", "0HB", "kp", "24Y", "0Er", "bDo", "aam", "5U0", "4Pp", "8bF", "Hl", "Gm", "10v", "5OP", "5Z1", +"anl", "45U", "0Js", "iA", "dq", "0GC", "48e", "6IN", "6gb", "4RA", "0io", "3Lm", "03e", "2BN", "7iA", "4LC", "4bo", "6WL", "zs", "0YA", +"0Tq", "3K", "a4f", "bUl", "4As", "5D3", "Yo", "87M", "01T", "Vn", "5K2", "4Nr", "54w", "417", "xB", "1KQ", "1Fa", "1z", "6XM", "4mn", +"4CB", "62i", "2MO", "0xl", "1za", "Ir", "6dM", "4Qn", "5oc", "6Ja", "25G", "67", "9Pe", "jn", "5w2", "46z", "bfm", "aCo", "J", "0gp", +"0hq", "KC", "72U", "bil", "5mR", "5x3", "eo", "326", "96", "3no", "7UA", "44K", "5NN", "6kL", "Fs", "0eA", "Xq", "1kb", "5PL", "61F", +"7KC", "4nA", "0Uo", "2U", "39U", "8QG", "4cq", "5f1", "aRl", "796", "0vs", "UA", "2LQ", "0yr", "767", "63w", "5i0", "4lp", "9Ng", "0d", +"2oM", "0Zn", "55i", "6Tc", "6zO", "4Ol", "00J", "Wp", "6FK", "4sh", "0HF", "kt", "EX", "P9", "5Me", "6hg", "5U4", "4Pt", "0kZ", "Hh", +"fD", "0Ev", "5ny", "4k9", "4d8", "45Q", "0Jw", "iE", "Gi", "10r", "5OT", "5Z5", "6gf", "4RE", "0ik", "JY", "du", "0GG", "48a", "6IJ", +"4bk", "6WH", "zw", "0YE", "03a", "2BJ", "6yd", "4LG", "4Aw", "5D7", "Yk", "0zY", "0Tu", "3O", "6Zx", "bUh", "54s", "413", "xF", "1KU", +"01P", "Vj", "5K6", "4Nv", "4CF", "62m", "2MK", "0xh", "0VD", "uv", "6XI", "4mj", "5NS", "6kQ", "Fn", "11u", "0Kp", "hB", "aoo", "44V", +"49f", "6HM", "er", "1Va", "0hl", "3Mn", "6fa", "4SB", "5Lb", "7yA", "W", "0gm", "0IA", "js", "6GL", "46g", "bEl", "hyW", "gC", "0Dq", +"8cE", "Io", "5T3", "4Qs", "5J1", "4Oq", "00W", "Wm", "yA", "0Zs", "55t", "404", "6YN", "4lm", "0WC", "0y", "2LL", "0yo", "4BA", "63j", +"6xc", "bws", "02f", "2CM", "2ma", "0XB", "4cl", "6VO", "a5e", "bTo", "0Ur", "2H", "Xl", "86N", "5PQ", "5E0", "dh", "0GZ", "5lU", "5y4", +"4G9", "4RX", "0iv", "JD", "Gt", "0dF", "5OI", "6jK", "6Dg", "45L", "81", "iX", "fY", "70", "5nd", "6Kf", "6eJ", "4Pi", "0kG", "Hu", +"EE", "0fw", "5Mx", "4H8", "5v5", "4su", "1Xz", "ki", "0VY", "1c", "5h7", "4mw", "5Sz", "62p", "2MV", "0xu", "01M", "Vw", "7ki", "4Nk", +"54n", "6Ud", "2nJ", "1KH", "0Th", "3R", "6Ze", "4oF", "4Aj", "60A", "Yv", "0zD", "0wt", "TF", "6yy", "4LZ", "4bv", "5g6", "zj", "0YX", +"0Kt", "hF", "6Ey", "44R", "5NW", "6kU", "Fj", "0eX", "0hh", "KZ", "6fe", "4SF", "49b", "6HI", "ev", "0FD", "0IE", "jw", "6GH", "46c", +"5Lf", "6id", "S", "0gi", "0jY", "Ik", "5T7", "4Qw", "5oz", "6Jx", "gG", "0Du", "yE", "0Zw", "4aY", "400", "5J5", "4Ou", "00S", "Wi", +"ZY", "O8", "4BE", "63n", "6YJ", "4li", "0WG", "tu", "2me", "0XF", "4ch", "6VK", "6xg", "4MD", "02b", "UX", "Xh", "3K9", "5PU", "5E4", +"7KZ", "4nX", "0Uv", "2L", "73V", "bho", "0ir", "1l2", "dl", "335", "48x", "5y0", "6Dc", "45H", "85", "3ol", "Gp", "0dB", "5OM", "6jO", +"6eN", "4Pm", "0kC", "Hq", "24D", "74", "bDr", "6Kb", "529", "47y", "8AG", "km", "EA", "0fs", "bgn", "aBl", "774", "62t", "199", "0xq", +"9Od", "1g", "5h3", "4ms", "54j", "7EA", "2nN", "1KL", "01I", "Vs", "7km", "4No", "4An", "60E", "Yr", "1ja", "0Tl", "3V", "6Za", "4oB", +"4br", "5g2", "zn", "8PD", "03x", "TB", "aSo", "785", "49n", "6HE", "ez", "0FH", "0hd", "KV", "6fi", "4SJ", "bdI", "6kY", "Ff", "0eT", +"0Kx", "hJ", "4e7", "4pV", "5ov", "4j6", "gK", "0Dy", "0jU", "Ig", "6dX", "5AZ", "5Lj", "6ih", "DW", "Q6", "0II", "28b", "6GD", "46o", +"6YF", "4le", "0WK", "0q", "ZU", "O4", "4BI", "63b", "5J9", "4Oy", "0tW", "We", "yI", "1JZ", "4aU", "4t4", "7KV", "4nT", "0Uz", "vH", +"Xd", "1kw", "5PY", "5E8", "6xk", "4MH", "02n", "UT", "2mi", "0XJ", "4cd", "6VG", "2Qm", "0dN", "5OA", "6jC", "6Do", "45D", "89", "iP", +"0R3", "0GR", "48t", "acM", "4G1", "4RP", "94O", "JL", "EM", "12V", "5Mp", "4H0", "525", "47u", "0HS", "ka", "fQ", "78", "5nl", "6Kn", +"6eB", "4Pa", "0kO", "3NM", "01E", "3PO", "7ka", "4Nc", "54f", "6Ul", "xS", "m2", "0VQ", "1k", "a6F", "59V", "4CS", "4V2", "195", "85m", +"03t", "TN", "4Y3", "4LR", "56W", "a9G", "zb", "0YP", "b3", "3Z", "6Zm", "4oN", "4Ab", "60I", "2Oo", "0zL", "1xA", "KR", "6fm", "4SN", +"49j", "6HA", "27g", "0FL", "8Bd", "hN", "4e3", "44Z", "bdM", "aAO", "Fb", "0eP", "0jQ", "Ic", "70u", "655", "5or", "4j2", "gO", "8Me", +"0IM", "28f", "7Wa", "46k", "5Ln", "6il", "DS", "Q2", "ZQ", "O0", "4BM", "63f", "6YB", "4la", "0WO", "0u", "yM", "8Sg", "4aQ", "408", +"aPL", "b1F", "0tS", "Wa", "0n3", "1ks", "bzO", "61W", "7KR", "4nP", "214", "2D", "2mm", "0XN", "57I", "6VC", "6xo", "4ML", "02j", "UP", +"6Dk", "4qH", "0Jf", "iT", "Gx", "0dJ", "5OE", "6jG", "4G5", "4RT", "0iz", "JH", "dd", "0GV", "48p", "5y8", "521", "47q", "0HW", "ke", +"EI", "12R", "5Mt", "4H4", "6eF", "4Pe", "0kK", "Hy", "fU", "s4", "5nh", "6Kj", "54b", "6Uh", "xW", "m6", "01A", "3PK", "7ke", "4Ng", +"4CW", "4V6", "191", "0xy", "0VU", "1o", "6XX", "59R", "4bz", "6WY", "zf", "0YT", "03p", "TJ", "4Y7", "4LV", "4Af", "60M", "Yz", "0zH", +"b7", "wV", "6Zi", "4oJ", "5H3", "4Ms", "02U", "Uo", "2mR", "0Xq", "57v", "426", "7Km", "4no", "0UA", "vs", "2NN", "1kL", "5Pb", "61h", +"6za", "4OB", "00d", "2AO", "yr", "1Ja", "4an", "6TM", "a7g", "58w", "0Wp", "0J", "Zn", "84L", "4Br", "5G2", "5LQ", "5Y0", "d", "13w", +"0Ir", "1L2", "amm", "46T", "5oM", "6JO", "gp", "0DB", "0jn", "3Ol", "6dc", "5Aa", "bdr", "6kb", "2PL", "0eo", "0KC", "hq", "6EN", "44e", +"49U", "abl", "eA", "0Fs", "8aG", "Km", "5V1", "4Sq", "1Dz", "3a", "5j5", "4ou", "4AY", "4T8", "YE", "0zw", "03O", "Tu", "6yJ", "4Li", +"4bE", "6Wf", "zY", "o8", "0Vj", "1P", "6Xg", "4mD", "4Ch", "62C", "2Me", "0xF", "0uv", "VD", "7kZ", "4NX", "5pU", "5e4", "xh", "3k9", +"fj", "0EX", "5nW", "6KU", "6ey", "4PZ", "0kt", "HF", "Ev", "0fD", "5MK", "6hI", "6Fe", "47N", "0Hh", "kZ", "26B", "52", "48O", "6Id", +"6gH", "4Rk", "0iE", "Jw", "GG", "0du", "5Oz", "6jx", "5t7", "4qw", "0JY", "ik", "2mV", "0Xu", "57r", "422", "5H7", "4Mw", "02Q", "Uk", +"2NJ", "1kH", "5Pf", "61l", "7Ki", "4nk", "0UE", "vw", "yv", "0ZD", "4aj", "6TI", "6ze", "4OF", "0th", "WZ", "Zj", "0yX", "4Bv", "5G6", +"6Yy", "4lZ", "0Wt", "0N", "0Iv", "jD", "4g9", "46P", "5LU", "5Y4", "Dh", "0gZ", "0jj", "IX", "6dg", "4QD", "5oI", "6JK", "gt", "0DF", +"0KG", "hu", "6EJ", "44a", "5Nd", "6kf", "FY", "S8", "1xz", "Ki", "5V5", "4Su", "49Q", "4h8", "eE", "0Fw", "756", "60v", "YA", "0zs", +"9Mf", "3e", "5j1", "4oq", "4bA", "6Wb", "2lL", "0Yo", "03K", "Tq", "6yN", "4Lm", "4Cl", "62G", "2Ma", "0xB", "0Vn", "1T", "6Xc", "59i", +"54Y", "5e0", "xl", "8RF", "01z", "1p2", "aQm", "b0g", "71T", "bjm", "0kp", "HB", "fn", "317", "5nS", "6KQ", "6Fa", "47J", "0Hl", "29G", +"Er", "12i", "5MO", "6hM", "6gL", "4Ro", "0iA", "Js", "26F", "56", "48K", "7YA", "5t3", "4qs", "8CE", "io", "GC", "0dq", "bel", "hYW", +"7Ke", "4ng", "0UI", "2s", "XW", "M6", "5Pj", "6uh", "6xX", "6m9", "0vU", "Ug", "2mZ", "0Xy", "4cW", "4v6", "4y7", "4lV", "0Wx", "0B", +"Zf", "0yT", "4Bz", "63Q", "6zi", "4OJ", "B7", "WV", "yz", "0ZH", "4af", "6TE", "5oE", "6JG", "gx", "0DJ", "0jf", "IT", "6dk", "4QH", +"5LY", "5Y8", "l", "0gV", "0Iz", "jH", "4g5", "4rT", "5mt", "4h4", "eI", "1VZ", "0hW", "Ke", "5V9", "4Sy", "5Nh", "6kj", "FU", "S4", +"0KK", "hy", "6EF", "44m", "03G", "2Bl", "6yB", "4La", "4bM", "6Wn", "zQ", "o0", "0TS", "3i", "a4D", "bUN", "4AQ", "4T0", "YM", "87o", +"01v", "VL", "7kR", "4NP", "54U", "hft", "0N3", "1Ks", "0Vb", "1X", "6Xo", "4mL", "5SA", "62K", "2Mm", "0xN", "2So", "0fL", "5MC", "6hA", +"6Fm", "47F", "1XA", "kR", "fb", "0EP", "bDM", "aaO", "4E3", "4PR", "8bd", "HN", "GO", "10T", "5Or", "4J2", "507", "45w", "0JQ", "ic", +"dS", "q2", "48G", "6Il", "73i", "4Rc", "0iM", "3LO", "XS", "M2", "5Pn", "61d", "7Ka", "4nc", "0UM", "2w", "39w", "8Qe", "4cS", "4v2", +"aRN", "b3D", "02Y", "Uc", "Zb", "0yP", "bxM", "63U", "4y3", "4lR", "236", "0F", "2oo", "0ZL", "4ab", "6TA", "6zm", "4ON", "B3", "WR", +"0jb", "IP", "6do", "4QL", "5oA", "6JC", "25e", "0DN", "9PG", "jL", "4g1", "46X", "686", "aCM", "h", "0gR", "0hS", "Ka", "72w", "677", +"49Y", "4h0", "eM", "8Og", "0KO", "3nM", "6EB", "44i", "5Nl", "6kn", "FQ", "S0", "4bI", "6Wj", "zU", "o4", "03C", "Ty", "6yF", "4Le", +"4AU", "4T4", "YI", "1jZ", "0TW", "3m", "5j9", "4oy", "54Q", "5e8", "xd", "1Kw", "01r", "VH", "7kV", "4NT", "4Cd", "62O", "2Mi", "0xJ", +"0Vf", "uT", "6Xk", "4mH", "6Fi", "47B", "0Hd", "kV", "Ez", "0fH", "5MG", "6hE", "4E7", "4PV", "0kx", "HJ", "ff", "0ET", "bDI", "6KY", +"503", "45s", "0JU", "ig", "GK", "0dy", "5Ov", "4J6", "6gD", "4Rg", "0iI", "3LK", "dW", "q6", "48C", "6Ih", "4Z2", "4OS", "00u", "WO", +"yc", "0ZQ", "55V", "hgw", "6Yl", "4lO", "a2", "tS", "2Ln", "0yM", "4Bc", "63H", "6xA", "4Mb", "02D", "2Co", "2mC", "n3", "4cN", "6Vm", +"a5G", "bTM", "0UP", "2j", "XN", "86l", "5Ps", "4U3", "5Nq", "4K1", "FL", "11W", "0KR", "3nP", "514", "44t", "49D", "6Ho", "eP", "49", +"0hN", "3ML", "6fC", "5CA", "aV1", "6iB", "u", "0gO", "0Ic", "jQ", "6Gn", "46E", "bEN", "hyu", "ga", "0DS", "8cg", "IM", "4D0", "4QQ", +"1FZ", "1A", "4x4", "4mU", "4Cy", "5F9", "0m6", "0xW", "C4", "VU", "7kK", "4NI", "54L", "6UF", "xy", "1Kj", "0TJ", "3p", "6ZG", "4od", +"4AH", "60c", "YT", "L5", "0wV", "Td", "5I8", "4Lx", "4bT", "4w5", "zH", "0Yz", "dJ", "0Gx", "5lw", "4i7", "6gY", "4Rz", "0iT", "Jf", +"GV", "R7", "5Ok", "6ji", "6DE", "45n", "0JH", "iz", "24b", "0EI", "5nF", "6KD", "6eh", "4PK", "0ke", "HW", "Eg", "0fU", "5MZ", "6hX", +"4f6", "4sW", "0Hy", "kK", "yg", "0ZU", "55R", "6TX", "4Z6", "4OW", "00q", "WK", "2Lj", "0yI", "4Bg", "63L", "6Yh", "4lK", "a6", "tW", +"2mG", "n7", "4cJ", "6Vi", "6xE", "4Mf", "0vH", "Uz", "XJ", "1kY", "5Pw", "4U7", "7Kx", "4nz", "0UT", "2n", "0KV", "hd", "510", "44p", +"5Nu", "4K5", "FH", "0ez", "0hJ", "Kx", "6fG", "4Sd", "5mi", "6Hk", "eT", "p5", "0Ig", "jU", "6Gj", "46A", "5LD", "6iF", "q", "0gK", +"1zZ", "II", "4D4", "4QU", "5oX", "5z9", "ge", "0DW", "byN", "62V", "0m2", "0xS", "225", "1E", "4x0", "4mQ", "54H", "6UB", "2nl", "1Kn", +"C0", "VQ", "7kO", "4NM", "4AL", "60g", "YP", "L1", "0TN", "3t", "6ZC", "ae0", "4bP", "439", "zL", "8Pf", "03Z", "0b3", "aSM", "b2G", +"73t", "664", "0iP", "Jb", "dN", "8Nd", "48Z", "4i3", "6DA", "45j", "0JL", "3oN", "GR", "R3", "5Oo", "6jm", "6el", "4PO", "0ka", "HS", +"24f", "0EM", "5nB", "aaR", "4f2", "4sS", "8Ae", "kO", "Ec", "0fQ", "695", "aBN", "6Yd", "4lG", "0Wi", "0S", "Zw", "0yE", "4Bk", "6wH", +"6zx", "buh", "0tu", "WG", "yk", "0ZY", "4aw", "5d7", "5k6", "4nv", "0UX", "2b", "XF", "1kU", "741", "61q", "6xI", "4Mj", "02L", "Uv", +"2mK", "0Xh", "4cF", "6Ve", "49L", "6Hg", "eX", "41", "0hF", "Kt", "6fK", "4Sh", "5Ny", "4K9", "FD", "0ev", "0KZ", "hh", "5u4", "4pt", +"5oT", "5z5", "gi", "1Tz", "0jw", "IE", "4D8", "4QY", "5LH", "6iJ", "Du", "0gG", "0Ik", "jY", "6Gf", "46M", "01g", "3Pm", "7kC", "4NA", +"54D", "6UN", "xq", "1Kb", "0Vs", "1I", "a6d", "59t", "4Cq", "5F1", "d3G", "85O", "03V", "Tl", "5I0", "4Lp", "56u", "435", "2lQ", "0Yr", +"0TB", "3x", "6ZO", "4ol", "5Qa", "60k", "2OM", "0zn", "2QO", "0dl", "5Oc", "6ja", "6DM", "45f", "1Za", "ir", "dB", "0Gp", "48V", "aco", +"5W2", "4Rr", "94m", "Jn", "Eo", "12t", "5MR", "5X3", "aln", "47W", "0Hq", "kC", "fs", "0EA", "5nN", "6KL", "71I", "4PC", "0km", "3No", +"Zs", "0yA", "4Bo", "63D", "7IA", "4lC", "0Wm", "0W", "yo", "8SE", "4as", "5d3", "aPn", "b1d", "00y", "WC", "XB", "1kQ", "745", "61u", +"5k2", "4nr", "9Le", "2f", "2mO", "0Xl", "4cB", "6Va", "6xM", "4Mn", "02H", "Ur", "0hB", "Kp", "6fO", "4Sl", "49H", "6Hc", "27E", "45", +"8BF", "hl", "518", "44x", "bdo", "aAm", "2PQ", "0er", "0js", "IA", "70W", "bkn", "5oP", "5z1", "gm", "304", "0Io", "28D", "6Gb", "46I", +"5LL", "6iN", "y", "0gC", "5pH", "6UJ", "xu", "1Kf", "C8", "VY", "7kG", "4NE", "4Cu", "5F5", "2Mx", "1hz", "0Vw", "1M", "4x8", "4mY", +"4bX", "431", "zD", "0Yv", "03R", "Th", "5I4", "4Lt", "4AD", "60o", "YX", "L9", "0TF", "wt", "6ZK", "4oh", "6DI", "45b", "0JD", "iv", +"GZ", "0dh", "5Og", "6je", "5W6", "4Rv", "0iX", "Jj", "dF", "0Gt", "48R", "6Iy", "6Fx", "47S", "0Hu", "kG", "Ek", "0fY", "5MV", "5X7", +"6ed", "4PG", "0ki", "3Nk", "fw", "0EE", "5nJ", "6KH", "356", "bo", "6OP", "4zs", "bnl", "75U", "LC", "0oq", "0bA", "As", "6lL", "4Yo", +"43K", "7RA", "2yN", "0Lm", "17", "22G", "6Ma", "4xB", "4Vn", "6cM", "Nr", "19i", "14Y", "CB", "aDo", "bam", "41z", "5p2", "mn", "8GD", +"7d", "8YF", "4kp", "5n0", "64w", "717", "1nS", "2KQ", "Pp", "07J", "4Hl", "69G", "6Sc", "52i", "1MO", "2hM", "5U", "0Ro", "4iA", "7LC", +"66F", "4Gm", "08K", "3YA", "RA", "0qs", "b4f", "aUl", "5a1", "4dq", "8VG", "8e", "6mV", "4Xu", "17r", "022", "nE", "0Mw", "42Q", "4c8", +"6NJ", "5kH", "1Pf", "cu", "MY", "X8", "4UE", "74O", "6og", "4ZD", "W9", "BX", "lt", "0OF", "4th", "6AK", "4l9", "4yX", "0Bv", "aD", +"Oh", "0lZ", "4Wt", "5R4", "4Iv", "5L6", "Qj", "06P", "1LU", "1Y4", "463", "4gZ", "4jj", "7Oh", "rv", "0QD", "1oI", "2JK", "65m", "4DF", +"4KG", "7nE", "2EJ", "04a", "1Nd", "2kf", "6PH", "4ek", "5xz", "492", "4O", "0Su", "09Q", "0h8", "5C7", "4Fw", "5Dz", "6ax", "LG", "0ou", +"0AY", "bk", "6OT", "4zw", "43O", "6Bd", "2yJ", "0Li", "0bE", "Aw", "6lH", "4Yk", "4Vj", "6cI", "Nv", "0mD", "13", "22C", "6Me", "4xF", +"4uv", "5p6", "mj", "0NX", "1pU", "CF", "6ny", "7k9", "4P9", "4EX", "1nW", "2KU", "sh", "0PZ", "4kt", "5n4", "6Sg", "4fD", "k9", "2hI", +"Pt", "07N", "4Hh", "69C", "66B", "4Gi", "08O", "2Id", "5Q", "d8", "4iE", "7LG", "5a5", "4du", "1Oz", "8a", "RE", "0qw", "4JY", "aUh", +"nA", "0Ms", "42U", "ail", "6mR", "4Xq", "17v", "026", "3Km", "0no", "4UA", "74K", "6NN", "5kL", "1Pb", "cq", "lp", "0OB", "40d", "6AO", +"6oc", "5Ja", "0an", "2TM", "Ol", "18w", "4Wp", "5R0", "afm", "bCo", "0Br", "1G2", "1LQ", "1Y0", "467", "53w", "4Ir", "5L2", "Qn", "06T", +"1oM", "2JO", "65i", "4DB", "4jn", "7Ol", "6z", "1Aa", "8WY", "2kb", "6PL", "4eo", "4KC", "7nA", "2EN", "04e", "09U", "d6E", "5C3", "4Fs", +"bRl", "496", "4K", "0Sq", "0bI", "2Wj", "6lD", "4Yg", "43C", "6Bh", "oW", "z6", "0AU", "bg", "6OX", "5jZ", "4TW", "4A6", "LK", "0oy", +"14Q", "CJ", "4N7", "5Kw", "41r", "542", "mf", "0NT", "u7", "22O", "6Mi", "4xJ", "4Vf", "6cE", "Nz", "0mH", "Px", "07B", "4Hd", "69O", +"6Sk", "4fH", "k5", "2hE", "7l", "0PV", "4kx", "5n8", "4P5", "4ET", "83j", "2KY", "RI", "05s", "4JU", "7oW", "5a9", "4dy", "1Ov", "8m", +"qU", "d4", "4iI", "7LK", "66N", "4Ge", "08C", "2Ih", "6NB", "a59", "1Pn", "21d", "MQ", "X0", "4UM", "74G", "79w", "bbN", "0cS", "0v2", +"nM", "8Dg", "42Y", "4c0", "4l1", "4yP", "8Kf", "aL", "0y3", "0lR", "636", "76v", "6oo", "4ZL", "W1", "BP", "2zm", "0ON", "40h", "6AC", +"4jb", "auS", "6v", "0QL", "I3", "2JC", "65e", "4DN", "b7E", "68U", "Qb", "06X", "286", "dSl", "4r3", "4gR", "4hS", "7MQ", "4G", "277", +"09Y", "0h0", "67T", "b8D", "4KO", "7nM", "SS", "F2", "1Nl", "9w", "azR", "4ec", "43G", "6Bl", "oS", "z2", "0bM", "2Wn", "78i", "4Yc", +"4TS", "4A2", "LO", "8fe", "0AQ", "bc", "aeN", "cPm", "41v", "546", "mb", "0NP", "14U", "CN", "4N3", "5Ks", "4Vb", "6cA", "2Xo", "0mL", +"u3", "22K", "6Mm", "4xN", "6So", "4fL", "k1", "2hA", "2Fm", "07F", "5XA", "69K", "4P1", "4EP", "83n", "d5f", "7h", "0PR", "bQO", "a0E", +"hbu", "50T", "1Or", "8i", "RM", "05w", "4JQ", "7oS", "66J", "4Ga", "08G", "2Il", "5Y", "d0", "4iM", "7LO", "MU", "X4", "4UI", "74C", +"6NF", "5kD", "1Pj", "cy", "nI", "2m9", "4vU", "4c4", "6mZ", "4Xy", "0cW", "0v6", "Od", "0lV", "4Wx", "5R8", "4l5", "4yT", "0Bz", "aH", +"lx", "0OJ", "40l", "6AG", "6ok", "4ZH", "W5", "BT", "I7", "2JG", "65a", "4DJ", "4jf", "7Od", "6r", "0QH", "1LY", "1Y8", "4r7", "4gV", +"4Iz", "68Q", "Qf", "0rT", "1mt", "0h4", "67P", "5VZ", "4hW", "7MU", "4C", "0Sy", "1Nh", "9s", "6PD", "4eg", "4KK", "7nI", "SW", "F6", +"8Je", "22V", "4m2", "4xS", "625", "77u", "Nc", "0mQ", "V2", "CS", "6nl", "5Kn", "41k", "7Pa", "3kO", "0NM", "0AL", "20g", "6OA", "4zb", +"4TN", "6am", "LR", "Y3", "0bP", "Ab", "78t", "bcM", "43Z", "4b3", "oN", "8Ed", "5D", "264", "4iP", "489", "66W", "b9G", "08Z", "0i3", +"RP", "G1", "4JL", "7oN", "6QC", "50I", "1Oo", "8t", "7u", "0PO", "4ka", "7Nc", "64f", "4EM", "H0", "963", "Pa", "0sS", "b6F", "69V", +"478", "4fQ", "295", "dRo", "4O4", "4ZU", "15R", "BI", "le", "0OW", "40q", "551", "6Lj", "4yI", "t4", "aU", "Oy", "0lK", "4We", "6bF", +"6mG", "4Xd", "0cJ", "2Vi", "nT", "0Mf", "4vH", "6Ck", "adI", "5kY", "1Pw", "cd", "MH", "0nz", "4UT", "7pV", "4KV", "7nT", "SJ", "04p", +"1Nu", "9n", "6PY", "4ez", "4hJ", "7MH", "pV", "e7", "1mi", "2Hk", "67M", "4Ff", "4Ig", "68L", "2Gj", "06A", "j6", "2iF", "6Rh", "4gK", +"5zZ", "7Oy", "6o", "0QU", "1oX", "1z9", "4Q6", "4DW", "5FZ", "6cX", "Ng", "0mU", "0Cy", "1F9", "4m6", "4xW", "41o", "7Pe", "3kK", "0NI", +"V6", "CW", "6nh", "5Kj", "4TJ", "6ai", "LV", "Y7", "0AH", "bz", "6OE", "4zf", "4wV", "4b7", "oJ", "0Lx", "0bT", "Af", "6lY", "4Yz", +"5B8", "4Gx", "1lw", "0i7", "qH", "0Rz", "4iT", "7LV", "6QG", "4dd", "1Ok", "8p", "RT", "G5", "4JH", "7oJ", "64b", "4EI", "H4", "2KD", +"7q", "0PK", "4ke", "7Ng", "4s4", "4fU", "1MZ", "2hX", "Pe", "0sW", "4Hy", "5M9", "la", "0OS", "40u", "555", "4O0", "4ZQ", "15V", "BM", +"2Yl", "0lO", "4Wa", "6bB", "6Ln", "4yM", "08", "aQ", "nP", "0Mb", "42D", "6Co", "6mC", "5HA", "0cN", "2Vm", "ML", "8gf", "4UP", "74Z", +"adM", "bAO", "1Ps", "0U3", "1Nq", "9j", "azO", "51W", "4KR", "7nP", "SN", "04t", "09D", "2Ho", "67I", "4Fb", "4hN", "7ML", "4Z", "e3", +"j2", "2iB", "6Rl", "4gO", "4Ic", "68H", "2Gn", "06E", "82m", "d4e", "4Q2", "4DS", "bPL", "a1F", "6k", "0QQ", "1pH", "2UJ", "6nd", "5Kf", +"41c", "7Pi", "mw", "0NE", "0Cu", "1F5", "6Mx", "5hz", "4Vw", "5S7", "Nk", "0mY", "0bX", "Aj", "6lU", "4Yv", "43R", "6By", "oF", "0Lt", +"0AD", "bv", "6OI", "4zj", "4TF", "6ae", "LZ", "0oh", "RX", "G9", "4JD", "7oF", "6QK", "4dh", "1Og", "2je", "5L", "0Rv", "4iX", "481", +"5B4", "4Gt", "08R", "2Iy", "Pi", "07S", "4Hu", "5M5", "470", "4fY", "1MV", "1X7", "su", "0PG", "4ki", "7Nk", "64n", "4EE", "H8", "2KH", +"6Lb", "4yA", "04", "23D", "Oq", "0lC", "4Wm", "6bN", "aEl", "c4G", "0as", "BA", "lm", "8FG", "40y", "559", "6NS", "5kQ", "345", "cl", +"1k2", "0nr", "boo", "74V", "6mO", "4Xl", "0cB", "2Va", "2xM", "0Mn", "42H", "6Cc", "4hB", "aws", "4V", "0Sl", "09H", "2Hc", "67E", "4Fn", +"b5e", "aTo", "SB", "04x", "8WD", "9f", "6PQ", "4er", "4js", "5o3", "6g", "8XE", "1oP", "1z1", "65t", "704", "4Io", "68D", "Qs", "06I", +"1LL", "2iN", "7BA", "4gC", "41g", "7Pm", "ms", "0NA", "14D", "2UN", "aDr", "5Kb", "4Vs", "5S3", "No", "19t", "0Cq", "1F1", "agn", "bBl", +"43V", "aho", "oB", "0Lp", "16u", "An", "6lQ", "4Yr", "4TB", "6aa", "2ZO", "0ol", "1Qa", "br", "6OM", "4zn", "6QO", "4dl", "1Oc", "8x", +"2DM", "05f", "5Za", "7oB", "5B0", "4Gp", "08V", "d7F", "5H", "0Rr", "bSo", "485", "474", "52t", "1MR", "1X3", "Pm", "07W", "4Hq", "5M1", +"64j", "4EA", "1nN", "2KL", "7y", "0PC", "4km", "7No", "Ou", "0lG", "4Wi", "6bJ", "6Lf", "4yE", "00", "aY", "li", "8FC", "4tu", "5q5", +"4O8", "4ZY", "0aw", "BE", "MD", "0nv", "4UX", "74R", "6NW", "5kU", "341", "ch", "nX", "0Mj", "42L", "6Cg", "6mK", "4Xh", "0cF", "2Ve", +"09L", "2Hg", "67A", "4Fj", "4hF", "7MD", "4R", "0Sh", "1Ny", "9b", "6PU", "4ev", "4KZ", "7nX", "SF", "0pt", "1oT", "1z5", "65p", "5Tz", +"4jw", "5o7", "6c", "0QY", "1LH", "2iJ", "6Rd", "4gG", "4Ik", "7li", "Qw", "06M", "7F", "246", "4kR", "7NP", "64U", "col", "1nq", "0k1", +"PR", "E3", "4HN", "69e", "6SA", "4fb", "1Mm", "2ho", "5w", "0RM", "4ic", "7La", "66d", "4GO", "J2", "2IB", "Rc", "05Y", "b4D", "aUN", +"4q2", "4dS", "8Ve", "8G", "8Hg", "bM", "4o0", "4zQ", "607", "75w", "La", "0oS", "T0", "AQ", "6ln", "4YM", "43i", "6BB", "2yl", "0LO", +"0CN", "22e", "6MC", "5hA", "4VL", "6co", "NP", "0mb", "1ps", "0u3", "aDM", "baO", "41X", "7PR", "mL", "8Gf", "4IT", "7lV", "QH", "06r", +"1Lw", "0I7", "5b8", "4gx", "4jH", "7OJ", "rT", "g5", "1ok", "2Ji", "65O", "4Dd", "4Ke", "7ng", "Sy", "04C", "h4", "2kD", "6Pj", "4eI", +"4hy", "5m9", "4m", "0SW", "09s", "2HX", "4S4", "4FU", "4M6", "4XW", "0cy", "1f9", "ng", "0MU", "42s", "573", "6Nh", "5kj", "v6", "cW", +"3KK", "0nI", "4Ug", "74m", "6oE", "4Zf", "0aH", "Bz", "lV", "y7", "40B", "6Ai", "582", "4yz", "0BT", "af", "OJ", "0lx", "4WV", "4B7", +"64Q", "4Ez", "1nu", "0k5", "7B", "0Px", "4kV", "7NT", "6SE", "4ff", "1Mi", "2hk", "PV", "E7", "4HJ", "69a", "6rh", "4GK", "J6", "2IF", +"5s", "0RI", "4ig", "7Le", "4q6", "4dW", "1OX", "8C", "Rg", "0qU", "5ZZ", "7oy", "4Ty", "5Q9", "Le", "0oW", "1QZ", "bI", "4o4", "4zU", +"43m", "6BF", "oy", "0LK", "T4", "AU", "6lj", "4YI", "4VH", "6ck", "NT", "0mf", "0CJ", "22a", "6MG", "4xd", "4uT", "7PV", "mH", "0Nz", +"1pw", "Cd", "aDI", "5KY", "1Ls", "0I3", "axM", "53U", "4IP", "7lR", "QL", "06v", "1oo", "2Jm", "65K", "5TA", "4jL", "7ON", "6X", "g1", +"h0", "9Y", "6Pn", "4eM", "4Ka", "7nc", "2El", "04G", "09w", "d6g", "4S0", "4FQ", "bRN", "a3D", "4i", "0SS", "nc", "0MQ", "42w", "577", +"4M2", "4XS", "17T", "dlm", "3KO", "0nM", "4Uc", "74i", "6Nl", "5kn", "v2", "cS", "lR", "y3", "40F", "6Am", "6oA", "4Zb", "0aL", "2To", +"ON", "18U", "4WR", "4B3", "586", "bCM", "0BP", "ab", "PZ", "0sh", "4HF", "69m", "6SI", "4fj", "1Me", "2hg", "7N", "0Pt", "4kZ", "7NX", +"6pU", "4Ev", "1ny", "0k9", "Rk", "05Q", "4Jw", "5O7", "452", "50r", "1OT", "8O", "qw", "0RE", "4ik", "7Li", "66l", "4GG", "08a", "2IJ", +"T8", "AY", "6lf", "4YE", "43a", "6BJ", "ou", "0LG", "0Aw", "bE", "4o8", "4zY", "4Tu", "5Q5", "Li", "8fC", "14s", "Ch", "6nW", "5KU", +"41P", "7PZ", "mD", "0Nv", "0CF", "22m", "6MK", "4xh", "4VD", "6cg", "NX", "0mj", "5za", "7OB", "6T", "0Qn", "1oc", "2Ja", "65G", "4Dl", +"b7g", "68w", "1w2", "06z", "8UF", "dSN", "5b0", "4gp", "4hq", "5m1", "4e", "8ZG", "1mR", "1x3", "67v", "726", "4Km", "7no", "Sq", "04K", +"1NN", "9U", "6Pb", "4eA", "adr", "5kb", "26", "21F", "Ms", "0nA", "4Uo", "74e", "79U", "bbl", "0cq", "1f1", "no", "396", "4vs", "5s3", +"6LQ", "4yr", "367", "an", "OB", "0lp", "bmm", "76T", "6oM", "4Zn", "15i", "Br", "2zO", "0Ol", "40J", "6Aa", "6SM", "4fn", "1Ma", "2hc", +"2FO", "07d", "4HB", "69i", "64Y", "4Er", "83L", "d5D", "7J", "0Pp", "bQm", "a0g", "456", "50v", "1OP", "8K", "Ro", "05U", "4Js", "5O3", +"66h", "4GC", "08e", "2IN", "qs", "0RA", "4io", "7Lm", "43e", "6BN", "oq", "0LC", "0bo", "2WL", "6lb", "4YA", "4Tq", "5Q1", "Lm", "8fG", +"0As", "bA", "ael", "cPO", "41T", "ajm", "1K2", "0Nr", "14w", "Cl", "6nS", "5KQ", "5Fa", "6cc", "2XM", "0mn", "0CB", "22i", "6MO", "4xl", +"1og", "2Je", "65C", "4Dh", "4jD", "7OF", "6P", "g9", "3l9", "2iy", "5b4", "4gt", "4IX", "68s", "QD", "0rv", "1mV", "1x7", "4S8", "4FY", +"4hu", "5m5", "4a", "1Cz", "h8", "9Q", "6Pf", "4eE", "4Ki", "7nk", "Su", "04O", "Mw", "0nE", "4Uk", "74a", "6Nd", "5kf", "22", "21B", +"nk", "0MY", "4vw", "5s7", "6mx", "5Hz", "0cu", "1f5", "OF", "0lt", "4WZ", "6by", "6LU", "4yv", "0BX", "aj", "lZ", "0Oh", "40N", "6Ae", +"6oI", "4Zj", "0aD", "Bv", "5f", "9Ke", "4ir", "5l2", "66u", "735", "08x", "1y0", "Rr", "05H", "4Jn", "7ol", "6Qa", "4dB", "1OM", "8V", +"7W", "0Pm", "4kC", "7NA", "64D", "4Eo", "83Q", "2Kb", "PC", "07y", "b6d", "69t", "5c3", "4fs", "8TE", "dRM", "374", "22t", "599", "4xq", +"bln", "77W", "NA", "0ms", "14j", "Cq", "6nN", "5KL", "41I", "7PC", "3km", "0No", "35", "20E", "6Oc", "5ja", "4Tl", "6aO", "Lp", "0oB", +"0br", "1g2", "78V", "bco", "43x", "568", "ol", "385", "4Kt", "5N4", "Sh", "04R", "1NW", "9L", "441", "4eX", "4hh", "7Mj", "pt", "0SF", +"K9", "2HI", "67o", "4FD", "4IE", "68n", "QY", "0", "1Lf", "2id", "6RJ", "4gi", "4jY", "auh", "6M", "0Qw", "1oz", "2Jx", "5A5", "4Du", +"6oT", "4Zw", "0aY", "Bk", "lG", "0Ou", "40S", "6Ax", "6LH", "4yk", "0BE", "aw", "2YJ", "0li", "4WG", "6bd", "6me", "4XF", "0ch", "2VK", +"nv", "0MD", "42b", "6CI", "6Ny", "7K9", "1PU", "cF", "Mj", "0nX", "4Uv", "5P6", "66q", "4GZ", "1lU", "1y4", "5b", "0RX", "4iv", "5l6", +"6Qe", "4dF", "1OI", "8R", "Rv", "05L", "4Jj", "7oh", "6pH", "4Ek", "1nd", "2Kf", "7S", "0Pi", "4kG", "7NE", "5c7", "4fw", "1Mx", "0H8", +"PG", "0su", "5Xz", "69p", "4VY", "4C8", "NE", "0mw", "1Sz", "22p", "6MV", "4xu", "41M", "7PG", "mY", "x8", "14n", "Cu", "6nJ", "5KH", +"4Th", "6aK", "Lt", "0oF", "31", "bX", "6Og", "4zD", "4wt", "5r4", "oh", "0LZ", "0bv", "AD", "4L9", "4YX", "1NS", "9H", "445", "51u", +"4Kp", "5N0", "Sl", "04V", "09f", "2HM", "67k", "5Va", "4hl", "7Mn", "4x", "0SB", "1Lb", "3yA", "6RN", "4gm", "4IA", "68j", "2GL", "4", +"82O", "d4G", "5A1", "4Dq", "bPn", "a1d", "6I", "0Qs", "lC", "0Oq", "40W", "akn", "6oP", "4Zs", "15t", "Bo", "2YN", "0lm", "4WC", "76I", +"6LL", "4yo", "0BA", "as", "nr", "8DX", "42f", "6CM", "6ma", "4XB", "0cl", "2VO", "Mn", "8gD", "4Ur", "5P2", "ado", "bAm", "1PQ", "cB", +"Rz", "0qH", "4Jf", "7od", "6Qi", "4dJ", "i7", "2jG", "5n", "0RT", "4iz", "7Lx", "4R7", "4GV", "08p", "1y8", "PK", "07q", "4HW", "7mU", +"6SX", "52R", "1Mt", "0H4", "sW", "f6", "4kK", "7NI", "64L", "4Eg", "1nh", "2Kj", "14b", "Cy", "6nF", "5KD", "41A", "7PK", "mU", "x4", +"0CW", "0V6", "591", "4xy", "4VU", "4C4", "NI", "19R", "0bz", "AH", "4L5", "4YT", "43p", "560", "od", "0LV", "w5", "bT", "6Ok", "4zH", +"4Td", "6aG", "Lx", "0oJ", "5xA", "7Mb", "4t", "0SN", "K1", "2HA", "67g", "4FL", "b5G", "aTM", "0e3", "04Z", "8Wf", "9D", "449", "4eP", +"4jQ", "7OS", "6E", "255", "1or", "0j2", "65V", "cno", "4IM", "68f", "QQ", "8", "1Ln", "2il", "6RB", "4ga", "afR", "4yc", "0BM", "23f", +"OS", "Z2", "4WO", "6bl", "aEN", "c4e", "0aQ", "Bc", "lO", "8Fe", "4tS", "4a2", "4n3", "5ks", "8Id", "cN", "Mb", "0nP", "614", "74t", +"6mm", "4XN", "U3", "2VC", "2xo", "0ML", "42j", "6CA", "6Qm", "4dN", "i3", "8Z", "2Do", "05D", "4Jb", "aUS", "4R3", "4GR", "08t", "d7d", +"5j", "0RP", "bSM", "a2G", "ayN", "52V", "1Mp", "0H0", "PO", "07u", "4HS", "69x", "64H", "4Ec", "1nl", "2Kn", "sS", "f2", "4kO", "7NM", +"41E", "7PO", "mQ", "x0", "14f", "2Ul", "6nB", "aQ1", "4VQ", "4C0", "NM", "19V", "0CS", "0V2", "595", "bBN", "43t", "564", "0Y3", "0LR", +"16W", "AL", "4L1", "4YP", "5DA", "6aC", "2Zm", "0oN", "39", "bP", "6Oo", "4zL", "K5", "2HE", "67c", "4FH", "4hd", "7Mf", "4p", "0SJ", +"8Wb", "2kY", "4p5", "4eT", "4Kx", "5N8", "Sd", "0pV", "1ov", "0j6", "5A9", "4Dy", "4jU", "7OW", "6A", "1AZ", "1Lj", "2ih", "6RF", "4ge", +"4II", "68b", "QU", "D4", "OW", "Z6", "4WK", "6bh", "6LD", "4yg", "0BI", "23b", "lK", "0Oy", "4tW", "4a6", "6oX", "5JZ", "0aU", "Bg", +"Mf", "0nT", "4Uz", "74p", "4n7", "5kw", "1PY", "cJ", "nz", "0MH", "42n", "6CE", "6mi", "4XJ", "U7", "2VG", "4MP", "4X1", "UL", "02v", +"0XR", "0M3", "a8E", "57U", "4nL", "7KN", "2X", "c1", "1ko", "2Nm", "61K", "5PA", "4Oa", "6zB", "2Al", "00G", "l0", "yQ", "6Tn", "4aM", +"58T", "a7D", "0i", "0WS", "84o", "ZM", "4W0", "4BQ", "4I2", "5Lr", "13T", "G", "jc", "0IQ", "46w", "537", "6Jl", "5on", "r2", "gS", +"3OO", "0jM", "4Qc", "70i", "6kA", "5NC", "0eL", "2Po", "hR", "8Bx", "44F", "6Em", "abO", "49v", "0FP", "eb", "KN", "8ad", "4SR", "4F3", +"3B", "0Tx", "4oV", "4z7", "60Q", "4Az", "0zT", "Yf", "TV", "A7", "4LJ", "6yi", "6WE", "4bf", "0YH", "zz", "1s", "0VI", "4mg", "6XD", +"6vh", "4CK", "N6", "2MF", "Vg", "0uU", "6n9", "7ky", "4u6", "5pv", "1KX", "xK", "1UZ", "fI", "4k4", "5nt", "4Py", "5U9", "He", "0kW", +"P4", "EU", "6hj", "5Mh", "47m", "6FF", "ky", "0HK", "0GJ", "dx", "6IG", "48l", "4RH", "6gk", "JT", "0if", "0dV", "Gd", "5Z8", "5OY", +"4qT", "4d5", "iH", "0Jz", "0XV", "0M7", "5f8", "4cx", "4MT", "4X5", "UH", "02r", "1kk", "Xx", "61O", "5PE", "4nH", "7KJ", "vT", "c5", +"l4", "yU", "6Tj", "4aI", "4Oe", "6zF", "Wy", "00C", "1iZ", "ZI", "4W4", "4BU", "4ly", "5i9", "0m", "0WW", "jg", "0IU", "46s", "533", +"4I6", "5Lv", "0gy", "C", "3OK", "0jI", "4Qg", "6dD", "6Jh", "5oj", "r6", "gW", "hV", "0Kd", "44B", "6Ei", "6kE", "5NG", "0eH", "Fz", +"KJ", "0hx", "4SV", "4F7", "6HY", "49r", "0FT", "ef", "60U", "ckl", "0zP", "Yb", "3F", "206", "4oR", "4z3", "6WA", "4bb", "0YL", "2lo", +"TR", "A3", "4LN", "6ym", "62d", "4CO", "N2", "2MB", "1w", "0VM", "4mc", "7Ha", "4u2", "54z", "8Re", "xO", "Vc", "01Y", "b0D", "aQN", +"647", "71w", "Ha", "0kS", "8Lg", "fM", "4k0", "5np", "47i", "6FB", "29d", "0HO", "P0", "EQ", "6hn", "5Ml", "4RL", "6go", "JP", "0ib", +"0GN", "26e", "6IC", "48h", "45X", "4d1", "iL", "8Cf", "0dR", "0q3", "hYt", "beO", "4nD", "7KF", "2P", "c9", "1kg", "Xt", "61C", "5PI", +"4MX", "4X9", "UD", "0vv", "0XZ", "2my", "5f4", "4ct", "4lu", "5i5", "0a", "1Gz", "0yw", "ZE", "4W8", "4BY", "4Oi", "6zJ", "Wu", "00O", +"l8", "yY", "6Tf", "4aE", "6Jd", "5of", "62", "25B", "Iw", "0jE", "4Qk", "6dH", "6ix", "5Lz", "0gu", "O", "jk", "0IY", "4rw", "5w7", +"5x6", "5mW", "0FX", "ej", "KF", "0ht", "4SZ", "6fy", "6kI", "5NK", "0eD", "Fv", "hZ", "93", "44N", "6Ee", "2BO", "03d", "4LB", "6ya", +"6WM", "4bn", "1Ia", "zr", "3J", "0Tp", "bUm", "a4g", "5D2", "4Ar", "87L", "Yn", "Vo", "01U", "4Ns", "5K3", "416", "54v", "1KP", "xC", +"us", "0VA", "4mo", "6XL", "62h", "4CC", "0xm", "2MN", "0fo", "2SL", "6hb", "bgr", "47e", "6FN", "kq", "0HC", "0Es", "fA", "aal", "bDn", +"4Pq", "5U1", "Hm", "8bG", "10w", "Gl", "5Z0", "5OQ", "45T", "anm", "1O2", "0Jr", "0GB", "dp", "6IO", "48d", "5Ba", "6gc", "3Ll", "0in", +"1kc", "Xp", "61G", "5PM", "bTs", "7KB", "2T", "0Un", "8QF", "39T", "5f0", "4cp", "797", "aRm", "1s2", "02z", "0ys", "ZA", "63v", "766", +"4lq", "5i1", "0e", "9Nf", "0Zo", "2oL", "6Tb", "4aA", "4Om", "6zN", "Wq", "00K", "Is", "0jA", "4Qo", "6dL", "7ZA", "5ob", "66", "25F", +"jo", "9Pd", "4rs", "5w3", "aCn", "bfl", "0gq", "K", "KB", "0hp", "bim", "72T", "5x2", "49z", "327", "en", "3nn", "97", "44J", "6Ea", +"6kM", "5NO", "11i", "Fr", "6WI", "4bj", "0YD", "zv", "TZ", "0wh", "4LF", "6ye", "5D6", "4Av", "0zX", "Yj", "3N", "0Tt", "4oZ", "6Zy", +"412", "54r", "1KT", "xG", "Vk", "01Q", "4Nw", "5K7", "62l", "4CG", "0xi", "2MJ", "uw", "0VE", "4mk", "6XH", "47a", "6FJ", "ku", "0HG", +"P8", "EY", "6hf", "5Md", "4Pu", "5U5", "Hi", "8bC", "0Ew", "fE", "4k8", "5nx", "45P", "4d9", "iD", "0Jv", "0dZ", "Gh", "5Z4", "5OU", +"4RD", "6gg", "JX", "0ij", "0GF", "dt", "6IK", "5lI", "4Op", "5J0", "Wl", "00V", "0Zr", "2oQ", "405", "55u", "4ll", "6YO", "0x", "0WB", +"0yn", "2LM", "63k", "5Ra", "4MA", "6xb", "2CL", "02g", "0XC", "39I", "6VN", "4cm", "bTn", "a5d", "2I", "0Us", "86O", "Xm", "5E1", "5PP", +"6kP", "5NR", "11t", "Fo", "hC", "0Kq", "44W", "aon", "6HL", "49g", "0FA", "es", "3Mo", "0hm", "4SC", "72I", "6ia", "5Lc", "0gl", "V", +"jr", "1Ya", "46f", "6GM", "hyV", "bEm", "0Dp", "gB", "In", "8cD", "4Qr", "5T2", "1b", "0VX", "4mv", "5h6", "62q", "4CZ", "0xt", "2MW", +"Vv", "01L", "4Nj", "7kh", "6Ue", "54o", "1KI", "xZ", "3S", "0Ti", "4oG", "6Zd", "6tH", "4Ak", "0zE", "Yw", "TG", "0wu", "780", "6yx", +"5g7", "4bw", "0YY", "zk", "1Wz", "di", "5y5", "5lT", "4RY", "4G8", "JE", "0iw", "0dG", "Gu", "6jJ", "5OH", "45M", "6Df", "iY", "80", +"71", "fX", "6Kg", "5ne", "4Ph", "6eK", "Ht", "0kF", "0fv", "ED", "4H9", "5My", "4st", "5v4", "kh", "0HZ", "0Zv", "yD", "401", "4aX", +"4Ot", "5J4", "Wh", "00R", "O9", "ZX", "63o", "4BD", "4lh", "6YK", "tt", "0WF", "0XG", "2md", "6VJ", "4ci", "4ME", "6xf", "UY", "02c", +"1kz", "Xi", "5E5", "5PT", "4nY", "aqh", "2M", "0Uw", "hG", "0Ku", "44S", "6Ex", "6kT", "5NV", "0eY", "Fk", "3Mk", "0hi", "4SG", "6fd", +"6HH", "49c", "0FE", "ew", "jv", "0ID", "46b", "6GI", "6ie", "5Lg", "0gh", "R", "Ij", "0jX", "4Qv", "5T6", "6Jy", "7O9", "0Dt", "gF", +"62u", "775", "0xp", "198", "1f", "9Oe", "4mr", "5h2", "6Ua", "54k", "1KM", "2nO", "Vr", "01H", "4Nn", "7kl", "60D", "4Ao", "0zA", "Ys", +"3W", "0Tm", "4oC", "7JA", "5g3", "4bs", "8PE", "zo", "TC", "03y", "784", "aSn", "bhn", "73W", "JA", "0is", "334", "dm", "5y1", "48y", +"45I", "6Db", "3om", "84", "0dC", "Gq", "6jN", "5OL", "4Pl", "6eO", "Hp", "0kB", "75", "24E", "6Kc", "5na", "47x", "528", "kl", "8AF", +"0fr", "1c2", "aBm", "bgo", "4ld", "6YG", "0p", "0WJ", "O5", "ZT", "63c", "4BH", "4Ox", "5J8", "Wd", "0tV", "0Zz", "yH", "4t5", "4aT", +"4nU", "7KW", "2A", "1EZ", "1kv", "Xe", "5E9", "5PX", "4MI", "6xj", "UU", "02o", "0XK", "2mh", "6VF", "4ce", "6HD", "49o", "0FI", "27b", +"KW", "0he", "4SK", "6fh", "6kX", "5NZ", "0eU", "Fg", "hK", "0Ky", "4pW", "4e6", "4j7", "5ow", "0Dx", "gJ", "If", "0jT", "4Qz", "6dY", +"6ii", "5Lk", "Q7", "DV", "jz", "0IH", "46n", "6GE", "3PN", "01D", "4Nb", "aQS", "6Um", "54g", "m3", "xR", "1j", "0VP", "59W", "a6G", +"4V3", "4CR", "85l", "194", "TO", "03u", "4LS", "4Y2", "a9F", "56V", "0YQ", "zc", "wS", "b2", "4oO", "6Zl", "60H", "4Ac", "0zM", "2On", +"0dO", "2Ql", "6jB", "aU1", "45E", "6Dn", "iQ", "88", "0GS", "da", "acL", "48u", "4RQ", "4G0", "JM", "94N", "12W", "EL", "4H1", "5Mq", +"47t", "524", "29y", "0HR", "79", "fP", "6Ko", "5nm", "aZ0", "6eC", "3NL", "0kN", "O1", "ZP", "63g", "4BL", "58I", "6YC", "0t", "0WN", +"8Sf", "yL", "409", "4aP", "b1G", "aPM", "0a3", "00Z", "1kr", "Xa", "61V", "bzN", "4nQ", "7KS", "2E", "215", "0XO", "2ml", "6VB", "4ca", +"4MM", "6xn", "UQ", "02k", "KS", "0ha", "4SO", "6fl", "7Xa", "49k", "0FM", "27f", "hO", "8Be", "4pS", "4e2", "aAN", "bdL", "0eQ", "Fc", +"Ib", "0jP", "654", "70t", "4j3", "5os", "8Md", "gN", "28g", "0IL", "46j", "6GA", "6im", "5Lo", "Q3", "Z", "6Ui", "54c", "m7", "xV", +"Vz", "0uH", "4Nf", "7kd", "4V7", "4CV", "0xx", "190", "1n", "0VT", "4mz", "6XY", "6WX", "56R", "0YU", "zg", "TK", "03q", "4LW", "4Y6", +"60L", "4Ag", "0zI", "2Oj", "wW", "b6", "4oK", "6Zh", "45A", "6Dj", "iU", "0Jg", "0dK", "Gy", "6jF", "5OD", "4RU", "4G4", "JI", "1yZ", +"0GW", "de", "5y9", "48q", "47p", "520", "kd", "0HV", "0fz", "EH", "4H5", "5Mu", "4Pd", "6eG", "Hx", "0kJ", "s5", "fT", "6Kk", "5ni", +"5Y1", "5LP", "13v", "e", "jA", "0Is", "46U", "aml", "6JN", "5oL", "0DC", "gq", "3Om", "0jo", "4QA", "6db", "6kc", "5Na", "0en", "2PM", +"hp", "0KB", "44d", "6EO", "abm", "49T", "0Fr", "1C2", "Kl", "8aF", "4Sp", "5V0", "4Mr", "5H2", "Un", "02T", "0Xp", "2mS", "427", "57w", +"4nn", "7Kl", "2z", "1Ea", "1kM", "2NO", "61i", "5Pc", "4OC", "7jA", "2AN", "00e", "0ZA", "ys", "6TL", "4ao", "58v", "a7f", "0K", "0Wq", +"84M", "Zo", "5G3", "4Bs", "0EY", "fk", "6KT", "5nV", "bjh", "6ex", "HG", "0ku", "0fE", "Ew", "6hH", "5MJ", "47O", "6Fd", "29B", "0Hi", +"53", "dZ", "6Ie", "48N", "4Rj", "6gI", "Jv", "0iD", "0dt", "GF", "6jy", "7o9", "4qv", "5t6", "ij", "0JX", "wh", "0TZ", "4ot", "5j4", +"4T9", "4AX", "0zv", "YD", "Tt", "03N", "4Lh", "6yK", "6Wg", "4bD", "o9", "zX", "1Q", "0Vk", "4mE", "6Xf", "62B", "4Ci", "0xG", "2Md", +"VE", "0uw", "4NY", "aQh", "5e5", "5pT", "1Kz", "xi", "jE", "0Iw", "46Q", "4g8", "5Y5", "5LT", "13r", "a", "IY", "0jk", "4QE", "6df", +"6JJ", "5oH", "0DG", "gu", "ht", "0KF", "4ph", "6EK", "6kg", "5Ne", "S9", "FX", "Kh", "0hZ", "4St", "5V4", "4h9", "49P", "0Fv", "eD", +"0Xt", "2mW", "423", "4cZ", "4Mv", "5H6", "Uj", "02P", "1kI", "XZ", "61m", "5Pg", "4nj", "7Kh", "vv", "0UD", "0ZE", "yw", "6TH", "4ak", +"4OG", "6zd", "2AJ", "00a", "0yY", "Zk", "5G7", "4Bw", "58r", "6Yx", "0O", "0Wu", "bjl", "71U", "HC", "0kq", "316", "fo", "6KP", "5nR", +"47K", "7VA", "29F", "0Hm", "0fA", "Es", "6hL", "5MN", "4Rn", "6gM", "Jr", "1ya", "57", "26G", "6Ia", "48J", "45z", "5t2", "in", "8CD", +"0dp", "GB", "hYV", "bem", "60w", "757", "0zr", "2OQ", "3d", "9Mg", "4op", "5j0", "6Wc", "56i", "0Yn", "2lM", "Tp", "03J", "4Ll", "6yO", +"62F", "4Cm", "0xC", "dwS", "1U", "0Vo", "4mA", "6Xb", "5e1", "54X", "8RG", "xm", "VA", "0us", "b0f", "aQl", "6JF", "5oD", "0DK", "gy", +"IU", "0jg", "4QI", "6dj", "5Y9", "5LX", "0gW", "m", "jI", "1YZ", "4rU", "4g4", "4h5", "5mu", "0Fz", "eH", "Kd", "0hV", "4Sx", "5V8", +"6kk", "5Ni", "S5", "FT", "hx", "0KJ", "44l", "6EG", "4nf", "7Kd", "2r", "0UH", "M7", "XV", "61a", "5Pk", "4Mz", "6xY", "Uf", "0vT", +"0Xx", "39r", "4v7", "4cV", "4lW", "4y6", "0C", "0Wy", "0yU", "Zg", "63P", "5RZ", "4OK", "6zh", "WW", "B6", "0ZI", "2oj", "6TD", "4ag", +"0fM", "2Sn", "7xa", "5MB", "47G", "6Fl", "kS", "0Ha", "0EQ", "fc", "aaN", "bDL", "4PS", "4E2", "HO", "8be", "10U", "GN", "4J3", "5Os", +"45v", "506", "ib", "0JP", "q3", "dR", "6Im", "48F", "4Rb", "6gA", "3LN", "0iL", "2Bm", "03F", "aF0", "6yC", "6Wo", "4bL", "o1", "zP", +"3h", "0TR", "bUO", "a4E", "4T1", "4AP", "87n", "YL", "VM", "01w", "4NQ", "7kS", "hfu", "54T", "1Kr", "xa", "1Y", "0Vc", "4mM", "6Xn", +"62J", "4Ca", "0xO", "2Ml", "IQ", "0jc", "4QM", "6dn", "6JB", "a19", "0DO", "25d", "jM", "9PF", "46Y", "4g0", "aCL", "687", "0gS", "i", +"3MP", "0hR", "676", "72v", "4h1", "49X", "8Of", "eL", "3nL", "0KN", "44h", "6EC", "6ko", "5Nm", "S1", "FP", "M3", "XR", "61e", "5Po", +"4nb", "aqS", "2v", "0UL", "8Qd", "39v", "4v3", "4cR", "b3E", "aRO", "Ub", "02X", "0yQ", "Zc", "63T", "bxL", "4lS", "4y2", "0G", "237", +"0ZM", "2on", "7Da", "4ac", "4OO", "6zl", "WS", "B2", "47C", "6Fh", "kW", "0He", "0fI", "2Sj", "6hD", "5MF", "4PW", "4E6", "HK", "0ky", +"0EU", "fg", "6KX", "5nZ", "45r", "502", "if", "0JT", "0dx", "GJ", "4J7", "5Ow", "4Rf", "6gE", "Jz", "0iH", "q7", "dV", "6Ii", "48B", +"6Wk", "4bH", "o5", "zT", "Tx", "03B", "4Ld", "6yG", "4T5", "4AT", "0zz", "YH", "3l", "0TV", "4ox", "5j8", "5e9", "54P", "1Kv", "xe", +"VI", "01s", "4NU", "7kW", "62N", "4Ce", "0xK", "2Mh", "uU", "0Vg", "4mI", "6Xj", "4K0", "5Np", "11V", "FM", "ha", "0KS", "44u", "515", +"6Hn", "49E", "48", "eQ", "3MM", "0hO", "4Sa", "6fB", "6iC", "5LA", "0gN", "t", "jP", "0Ib", "46D", "6Go", "hyt", "bEO", "0DR", "0Q3", +"IL", "8cf", "4QP", "4D1", "4OR", "4Z3", "WN", "00t", "0ZP", "yb", "hgv", "55W", "4lN", "6Ym", "0Z", "a3", "0yL", "2Lo", "63I", "4Bb", +"4Mc", "7ha", "2Cn", "02E", "n2", "2mB", "6Vl", "4cO", "bTL", "a5F", "2k", "0UQ", "86m", "XO", "4U2", "5Pr", "0Gy", "dK", "4i6", "5lv", +"5BZ", "6gX", "Jg", "0iU", "R6", "GW", "6jh", "5Oj", "45o", "6DD", "3oK", "0JI", "0EH", "fz", "6KE", "5nG", "4PJ", "6ei", "HV", "0kd", +"0fT", "Ef", "6hY", "690", "4sV", "4f7", "kJ", "0Hx", "uH", "0Vz", "4mT", "4x5", "5F8", "4Cx", "0xV", "0m7", "VT", "C5", "4NH", "7kJ", +"6UG", "54M", "1Kk", "xx", "3q", "0TK", "4oe", "6ZF", "60b", "4AI", "L4", "YU", "Te", "0wW", "4Ly", "5I9", "4w4", "4bU", "1IZ", "zI", +"he", "0KW", "44q", "511", "4K4", "5Nt", "11R", "FI", "Ky", "0hK", "4Se", "6fF", "6Hj", "49A", "p4", "eU", "jT", "0If", "4rH", "6Gk", +"6iG", "5LE", "0gJ", "p", "IH", "0jz", "4QT", "4D5", "5z8", "5oY", "0DV", "gd", "0ZT", "yf", "6TY", "4az", "4OV", "4Z7", "WJ", "00p", +"0yH", "Zz", "63M", "4Bf", "4lJ", "6Yi", "tV", "a7", "n6", "2mF", "6Vh", "4cK", "4Mg", "6xD", "2Cj", "02A", "1kX", "XK", "4U6", "5Pv", +"6N9", "7Ky", "2o", "0UU", "665", "73u", "Jc", "0iQ", "8Ne", "dO", "4i2", "5lr", "45k", "7Ta", "3oO", "0JM", "R2", "GS", "6jl", "5On", +"4PN", "6em", "HR", "8bx", "0EL", "24g", "6KA", "5nC", "47Z", "4f3", "kN", "8Ad", "0fP", "Eb", "aBO", "694", "62W", "byO", "0xR", "0m3", +"1D", "224", "4mP", "4x1", "6UC", "54I", "1Ko", "2nm", "VP", "C1", "4NL", "7kN", "60f", "4AM", "L0", "YQ", "3u", "0TO", "4oa", "6ZB", +"438", "4bQ", "8Pg", "zM", "Ta", "0wS", "b2F", "aSL", "6Hf", "49M", "40", "eY", "Ku", "0hG", "4Si", "6fJ", "4K8", "5Nx", "0ew", "FE", +"hi", "8BC", "4pu", "5u5", "5z4", "5oU", "0DZ", "gh", "ID", "0jv", "4QX", "4D9", "6iK", "5LI", "0gF", "Dt", "jX", "0Ij", "46L", "6Gg", +"4lF", "6Ye", "0R", "0Wh", "0yD", "Zv", "63A", "4Bj", "4OZ", "6zy", "WF", "0tt", "0ZX", "yj", "5d6", "4av", "4nw", "5k7", "2c", "0UY", +"1kT", "XG", "61p", "5Pz", "4Mk", "6xH", "Uw", "02M", "0Xi", "2mJ", "6Vd", "4cG", "0dm", "2QN", "7zA", "5Ob", "45g", "6DL", "is", "0JA", +"0Gq", "dC", "acn", "48W", "4Rs", "5W3", "Jo", "94l", "12u", "En", "5X2", "5MS", "47V", "alo", "kB", "0Hp", "1Ua", "fr", "6KM", "5nO", +"4PB", "6ea", "3Nn", "0kl", "3Pl", "01f", "bts", "7kB", "6UO", "54E", "1Kc", "xp", "1H", "0Vr", "59u", "a6e", "5F0", "4Cp", "85N", "d3F", +"Tm", "03W", "4Lq", "5I1", "434", "56t", "0Ys", "zA", "3y", "0TC", "4om", "6ZN", "60j", "4AA", "0zo", "2OL", "Kq", "0hC", "4Sm", "6fN", +"6Hb", "49I", "44", "27D", "hm", "8BG", "44y", "519", "aAl", "bdn", "0es", "FA", "1o2", "0jr", "bko", "70V", "5z0", "5oQ", "305", "gl", +"28E", "0In", "46H", "6Gc", "6iO", "5LM", "0gB", "x", "1ia", "Zr", "63E", "4Bn", "4lB", "6Ya", "0V", "0Wl", "8SD", "yn", "5d2", "4ar", +"b1e", "aPo", "WB", "00x", "1kP", "XC", "61t", "744", "4ns", "5k3", "2g", "9Ld", "0Xm", "2mN", "7FA", "4cC", "4Mo", "6xL", "Us", "02I", +"45c", "6DH", "iw", "0JE", "0di", "2QJ", "6jd", "5Of", "4Rw", "5W7", "Jk", "0iY", "0Gu", "dG", "6Ix", "48S", "47R", "6Fy", "kF", "0Ht", +"0fX", "Ej", "5X6", "5MW", "4PF", "6ee", "HZ", "0kh", "0ED", "fv", "6KI", "5nK", "6UK", "54A", "1Kg", "xt", "VX", "C9", "4ND", "7kF", +"5F4", "4Ct", "0xZ", "2My", "1L", "0Vv", "4mX", "4x9", "430", "4bY", "0Yw", "zE", "Ti", "03S", "4Lu", "5I5", "60n", "4AE", "L8", "YY", +"wu", "0TG", "4oi", "6ZJ" }; + + +#endif + diff --git a/redis.submodule/src/crc64.c b/redis.submodule/src/crc64.c index f1f7649..4cbc019 100644 --- a/redis.submodule/src/crc64.c +++ b/redis.submodule/src/crc64.c @@ -1,16 +1,5 @@ -/* Redis uses the CRC64 variant with "Jones" coefficients and init value of 0. - * - * Specification of this CRC64 variant follows: - * Name: crc-64-jones - * Width: 64 bites - * Poly: 0xad93d23594c935a9 - * Reflected In: True - * Xor_In: 0xffffffffffffffff - * Reflected_Out: True - * Xor_Out: 0x0 - * Check("123456789"): 0xe9c6d914c4b8d9ca - * - * Copyright (c) 2012, Salvatore Sanfilippo +/* Copyright (c) 2014, Matt Stancliff + * Copyright (c) 2020, Amazon Web Services * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -37,147 +26,100 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ -#include - -static const uint64_t crc64_tab[256] = { - UINT64_C(0x0000000000000000), UINT64_C(0x7ad870c830358979), - UINT64_C(0xf5b0e190606b12f2), UINT64_C(0x8f689158505e9b8b), - UINT64_C(0xc038e5739841b68f), UINT64_C(0xbae095bba8743ff6), - UINT64_C(0x358804e3f82aa47d), UINT64_C(0x4f50742bc81f2d04), - UINT64_C(0xab28ecb46814fe75), UINT64_C(0xd1f09c7c5821770c), - UINT64_C(0x5e980d24087fec87), UINT64_C(0x24407dec384a65fe), - UINT64_C(0x6b1009c7f05548fa), UINT64_C(0x11c8790fc060c183), - UINT64_C(0x9ea0e857903e5a08), UINT64_C(0xe478989fa00bd371), - UINT64_C(0x7d08ff3b88be6f81), UINT64_C(0x07d08ff3b88be6f8), - UINT64_C(0x88b81eabe8d57d73), UINT64_C(0xf2606e63d8e0f40a), - UINT64_C(0xbd301a4810ffd90e), UINT64_C(0xc7e86a8020ca5077), - UINT64_C(0x4880fbd87094cbfc), UINT64_C(0x32588b1040a14285), - UINT64_C(0xd620138fe0aa91f4), UINT64_C(0xacf86347d09f188d), - UINT64_C(0x2390f21f80c18306), UINT64_C(0x594882d7b0f40a7f), - UINT64_C(0x1618f6fc78eb277b), UINT64_C(0x6cc0863448deae02), - UINT64_C(0xe3a8176c18803589), UINT64_C(0x997067a428b5bcf0), - UINT64_C(0xfa11fe77117cdf02), UINT64_C(0x80c98ebf2149567b), - UINT64_C(0x0fa11fe77117cdf0), UINT64_C(0x75796f2f41224489), - UINT64_C(0x3a291b04893d698d), UINT64_C(0x40f16bccb908e0f4), - UINT64_C(0xcf99fa94e9567b7f), UINT64_C(0xb5418a5cd963f206), - UINT64_C(0x513912c379682177), UINT64_C(0x2be1620b495da80e), - UINT64_C(0xa489f35319033385), UINT64_C(0xde51839b2936bafc), - UINT64_C(0x9101f7b0e12997f8), UINT64_C(0xebd98778d11c1e81), - UINT64_C(0x64b116208142850a), UINT64_C(0x1e6966e8b1770c73), - UINT64_C(0x8719014c99c2b083), UINT64_C(0xfdc17184a9f739fa), - UINT64_C(0x72a9e0dcf9a9a271), UINT64_C(0x08719014c99c2b08), - UINT64_C(0x4721e43f0183060c), UINT64_C(0x3df994f731b68f75), - UINT64_C(0xb29105af61e814fe), UINT64_C(0xc849756751dd9d87), - UINT64_C(0x2c31edf8f1d64ef6), UINT64_C(0x56e99d30c1e3c78f), - UINT64_C(0xd9810c6891bd5c04), UINT64_C(0xa3597ca0a188d57d), - UINT64_C(0xec09088b6997f879), UINT64_C(0x96d1784359a27100), - UINT64_C(0x19b9e91b09fcea8b), UINT64_C(0x636199d339c963f2), - UINT64_C(0xdf7adabd7a6e2d6f), UINT64_C(0xa5a2aa754a5ba416), - UINT64_C(0x2aca3b2d1a053f9d), UINT64_C(0x50124be52a30b6e4), - UINT64_C(0x1f423fcee22f9be0), UINT64_C(0x659a4f06d21a1299), - UINT64_C(0xeaf2de5e82448912), UINT64_C(0x902aae96b271006b), - UINT64_C(0x74523609127ad31a), UINT64_C(0x0e8a46c1224f5a63), - UINT64_C(0x81e2d7997211c1e8), UINT64_C(0xfb3aa75142244891), - UINT64_C(0xb46ad37a8a3b6595), UINT64_C(0xceb2a3b2ba0eecec), - UINT64_C(0x41da32eaea507767), UINT64_C(0x3b024222da65fe1e), - UINT64_C(0xa2722586f2d042ee), UINT64_C(0xd8aa554ec2e5cb97), - UINT64_C(0x57c2c41692bb501c), UINT64_C(0x2d1ab4dea28ed965), - UINT64_C(0x624ac0f56a91f461), UINT64_C(0x1892b03d5aa47d18), - UINT64_C(0x97fa21650afae693), UINT64_C(0xed2251ad3acf6fea), - UINT64_C(0x095ac9329ac4bc9b), UINT64_C(0x7382b9faaaf135e2), - UINT64_C(0xfcea28a2faafae69), UINT64_C(0x8632586aca9a2710), - UINT64_C(0xc9622c4102850a14), UINT64_C(0xb3ba5c8932b0836d), - UINT64_C(0x3cd2cdd162ee18e6), UINT64_C(0x460abd1952db919f), - UINT64_C(0x256b24ca6b12f26d), UINT64_C(0x5fb354025b277b14), - UINT64_C(0xd0dbc55a0b79e09f), UINT64_C(0xaa03b5923b4c69e6), - UINT64_C(0xe553c1b9f35344e2), UINT64_C(0x9f8bb171c366cd9b), - UINT64_C(0x10e3202993385610), UINT64_C(0x6a3b50e1a30ddf69), - UINT64_C(0x8e43c87e03060c18), UINT64_C(0xf49bb8b633338561), - UINT64_C(0x7bf329ee636d1eea), UINT64_C(0x012b592653589793), - UINT64_C(0x4e7b2d0d9b47ba97), UINT64_C(0x34a35dc5ab7233ee), - UINT64_C(0xbbcbcc9dfb2ca865), UINT64_C(0xc113bc55cb19211c), - UINT64_C(0x5863dbf1e3ac9dec), UINT64_C(0x22bbab39d3991495), - UINT64_C(0xadd33a6183c78f1e), UINT64_C(0xd70b4aa9b3f20667), - UINT64_C(0x985b3e827bed2b63), UINT64_C(0xe2834e4a4bd8a21a), - UINT64_C(0x6debdf121b863991), UINT64_C(0x1733afda2bb3b0e8), - UINT64_C(0xf34b37458bb86399), UINT64_C(0x8993478dbb8deae0), - UINT64_C(0x06fbd6d5ebd3716b), UINT64_C(0x7c23a61ddbe6f812), - UINT64_C(0x3373d23613f9d516), UINT64_C(0x49aba2fe23cc5c6f), - UINT64_C(0xc6c333a67392c7e4), UINT64_C(0xbc1b436e43a74e9d), - UINT64_C(0x95ac9329ac4bc9b5), UINT64_C(0xef74e3e19c7e40cc), - UINT64_C(0x601c72b9cc20db47), UINT64_C(0x1ac40271fc15523e), - UINT64_C(0x5594765a340a7f3a), UINT64_C(0x2f4c0692043ff643), - UINT64_C(0xa02497ca54616dc8), UINT64_C(0xdafce7026454e4b1), - UINT64_C(0x3e847f9dc45f37c0), UINT64_C(0x445c0f55f46abeb9), - UINT64_C(0xcb349e0da4342532), UINT64_C(0xb1eceec59401ac4b), - UINT64_C(0xfebc9aee5c1e814f), UINT64_C(0x8464ea266c2b0836), - UINT64_C(0x0b0c7b7e3c7593bd), UINT64_C(0x71d40bb60c401ac4), - UINT64_C(0xe8a46c1224f5a634), UINT64_C(0x927c1cda14c02f4d), - UINT64_C(0x1d148d82449eb4c6), UINT64_C(0x67ccfd4a74ab3dbf), - UINT64_C(0x289c8961bcb410bb), UINT64_C(0x5244f9a98c8199c2), - UINT64_C(0xdd2c68f1dcdf0249), UINT64_C(0xa7f41839ecea8b30), - UINT64_C(0x438c80a64ce15841), UINT64_C(0x3954f06e7cd4d138), - UINT64_C(0xb63c61362c8a4ab3), UINT64_C(0xcce411fe1cbfc3ca), - UINT64_C(0x83b465d5d4a0eece), UINT64_C(0xf96c151de49567b7), - UINT64_C(0x76048445b4cbfc3c), UINT64_C(0x0cdcf48d84fe7545), - UINT64_C(0x6fbd6d5ebd3716b7), UINT64_C(0x15651d968d029fce), - UINT64_C(0x9a0d8ccedd5c0445), UINT64_C(0xe0d5fc06ed698d3c), - UINT64_C(0xaf85882d2576a038), UINT64_C(0xd55df8e515432941), - UINT64_C(0x5a3569bd451db2ca), UINT64_C(0x20ed197575283bb3), - UINT64_C(0xc49581ead523e8c2), UINT64_C(0xbe4df122e51661bb), - UINT64_C(0x3125607ab548fa30), UINT64_C(0x4bfd10b2857d7349), - UINT64_C(0x04ad64994d625e4d), UINT64_C(0x7e7514517d57d734), - UINT64_C(0xf11d85092d094cbf), UINT64_C(0x8bc5f5c11d3cc5c6), - UINT64_C(0x12b5926535897936), UINT64_C(0x686de2ad05bcf04f), - UINT64_C(0xe70573f555e26bc4), UINT64_C(0x9ddd033d65d7e2bd), - UINT64_C(0xd28d7716adc8cfb9), UINT64_C(0xa85507de9dfd46c0), - UINT64_C(0x273d9686cda3dd4b), UINT64_C(0x5de5e64efd965432), - UINT64_C(0xb99d7ed15d9d8743), UINT64_C(0xc3450e196da80e3a), - UINT64_C(0x4c2d9f413df695b1), UINT64_C(0x36f5ef890dc31cc8), - UINT64_C(0x79a59ba2c5dc31cc), UINT64_C(0x037deb6af5e9b8b5), - UINT64_C(0x8c157a32a5b7233e), UINT64_C(0xf6cd0afa9582aa47), - UINT64_C(0x4ad64994d625e4da), UINT64_C(0x300e395ce6106da3), - UINT64_C(0xbf66a804b64ef628), UINT64_C(0xc5bed8cc867b7f51), - UINT64_C(0x8aeeace74e645255), UINT64_C(0xf036dc2f7e51db2c), - UINT64_C(0x7f5e4d772e0f40a7), UINT64_C(0x05863dbf1e3ac9de), - UINT64_C(0xe1fea520be311aaf), UINT64_C(0x9b26d5e88e0493d6), - UINT64_C(0x144e44b0de5a085d), UINT64_C(0x6e963478ee6f8124), - UINT64_C(0x21c640532670ac20), UINT64_C(0x5b1e309b16452559), - UINT64_C(0xd476a1c3461bbed2), UINT64_C(0xaeaed10b762e37ab), - UINT64_C(0x37deb6af5e9b8b5b), UINT64_C(0x4d06c6676eae0222), - UINT64_C(0xc26e573f3ef099a9), UINT64_C(0xb8b627f70ec510d0), - UINT64_C(0xf7e653dcc6da3dd4), UINT64_C(0x8d3e2314f6efb4ad), - UINT64_C(0x0256b24ca6b12f26), UINT64_C(0x788ec2849684a65f), - UINT64_C(0x9cf65a1b368f752e), UINT64_C(0xe62e2ad306bafc57), - UINT64_C(0x6946bb8b56e467dc), UINT64_C(0x139ecb4366d1eea5), - UINT64_C(0x5ccebf68aecec3a1), UINT64_C(0x2616cfa09efb4ad8), - UINT64_C(0xa97e5ef8cea5d153), UINT64_C(0xd3a62e30fe90582a), - UINT64_C(0xb0c7b7e3c7593bd8), UINT64_C(0xca1fc72bf76cb2a1), - UINT64_C(0x45775673a732292a), UINT64_C(0x3faf26bb9707a053), - UINT64_C(0x70ff52905f188d57), UINT64_C(0x0a2722586f2d042e), - UINT64_C(0x854fb3003f739fa5), UINT64_C(0xff97c3c80f4616dc), - UINT64_C(0x1bef5b57af4dc5ad), UINT64_C(0x61372b9f9f784cd4), - UINT64_C(0xee5fbac7cf26d75f), UINT64_C(0x9487ca0fff135e26), - UINT64_C(0xdbd7be24370c7322), UINT64_C(0xa10fceec0739fa5b), - UINT64_C(0x2e675fb4576761d0), UINT64_C(0x54bf2f7c6752e8a9), - UINT64_C(0xcdcf48d84fe75459), UINT64_C(0xb71738107fd2dd20), - UINT64_C(0x387fa9482f8c46ab), UINT64_C(0x42a7d9801fb9cfd2), - UINT64_C(0x0df7adabd7a6e2d6), UINT64_C(0x772fdd63e7936baf), - UINT64_C(0xf8474c3bb7cdf024), UINT64_C(0x829f3cf387f8795d), - UINT64_C(0x66e7a46c27f3aa2c), UINT64_C(0x1c3fd4a417c62355), - UINT64_C(0x935745fc4798b8de), UINT64_C(0xe98f353477ad31a7), - UINT64_C(0xa6df411fbfb21ca3), UINT64_C(0xdc0731d78f8795da), - UINT64_C(0x536fa08fdfd90e51), UINT64_C(0x29b7d047efec8728), -}; +#include "crc64.h" +#include "crcspeed.h" +static uint64_t crc64_table[8][256] = {{0}}; -uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) { - uint64_t j; +#define POLY UINT64_C(0xad93d23594c935a9) +/******************** BEGIN GENERATED PYCRC FUNCTIONS ********************/ +/** + * Generated on Sun Dec 21 14:14:07 2014, + * by pycrc v0.8.2, https://www.tty1.net/pycrc/ + * + * LICENSE ON GENERATED CODE: + * ========================== + * As of version 0.6, pycrc is released under the terms of the MIT licence. + * The code generated by pycrc is not considered a substantial portion of the + * software, therefore the author of pycrc will not claim any copyright on + * the generated code. + * ========================== + * + * CRC configuration: + * Width = 64 + * Poly = 0xad93d23594c935a9 + * XorIn = 0xffffffffffffffff + * ReflectIn = True + * XorOut = 0x0000000000000000 + * ReflectOut = True + * Algorithm = bit-by-bit-fast + * + * Modifications after generation (by matt): + * - included finalize step in-line with update for single-call generation + * - re-worked some inner variable architectures + * - adjusted function parameters to match expected prototypes. + *****************************************************************************/ + +/** + * Reflect all bits of a \a data word of \a data_len bytes. + * + * \param data The data word to be reflected. + * \param data_len The width of \a data expressed in number of bits. + * \return The reflected data. + *****************************************************************************/ +static inline uint_fast64_t crc_reflect(uint_fast64_t data, size_t data_len) { + uint_fast64_t ret = data & 0x01; + + for (size_t i = 1; i < data_len; i++) { + data >>= 1; + ret = (ret << 1) | (data & 0x01); + } + + return ret; +} + +/** + * Update the crc value with new data. + * + * \param crc The current crc value. + * \param data Pointer to a buffer of \a data_len bytes. + * \param data_len Number of bytes in the \a data buffer. + * \return The updated crc value. + ******************************************************************************/ +uint64_t _crc64(uint_fast64_t crc, const void *in_data, const uint64_t len) { + const uint8_t *data = in_data; + unsigned long long bit; - for (j = 0; j < l; j++) { - uint8_t byte = s[j]; - crc = crc64_tab[(uint8_t)crc ^ byte] ^ (crc >> 8); + for (uint64_t offset = 0; offset < len; offset++) { + uint8_t c = data[offset]; + for (uint_fast8_t i = 0x01; i & 0xff; i <<= 1) { + bit = crc & 0x8000000000000000; + if (c & i) { + bit = !bit; + } + + crc <<= 1; + if (bit) { + crc ^= POLY; + } + } + + crc &= 0xffffffffffffffff; } - return crc; + + crc = crc & 0xffffffffffffffff; + return crc_reflect(crc, 64) ^ 0x0000000000000000; +} + +/******************** END GENERATED PYCRC FUNCTIONS ********************/ + +/* Initializes the 16KB lookup tables. */ +void crc64_init(void) { + crcspeed64native_init(_crc64, crc64_table); +} + +/* Compute crc64 */ +uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) { + return crcspeed64native(crc64_table, crc, (void *) s, l); } /* Test main */ @@ -188,8 +130,31 @@ uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) { int crc64Test(int argc, char *argv[]) { UNUSED(argc); UNUSED(argv); - printf("e9c6d914c4b8d9ca == %016llx\n", - (unsigned long long) crc64(0,(unsigned char*)"123456789",9)); + crc64_init(); + printf("[calcula]: e9c6d914c4b8d9ca == %016" PRIx64 "\n", + (uint64_t)_crc64(0, "123456789", 9)); + printf("[64speed]: e9c6d914c4b8d9ca == %016" PRIx64 "\n", + (uint64_t)crc64(0, "123456789", 9)); + char li[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed " + "do eiusmod tempor incididunt ut labore et dolore magna " + "aliqua. Ut enim ad minim veniam, quis nostrud exercitation " + "ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis " + "aute irure dolor in reprehenderit in voluptate velit esse " + "cillum dolore eu fugiat nulla pariatur. Excepteur sint " + "occaecat cupidatat non proident, sunt in culpa qui officia " + "deserunt mollit anim id est laborum."; + printf("[calcula]: c7794709e69683b3 == %016" PRIx64 "\n", + (uint64_t)_crc64(0, li, sizeof(li))); + printf("[64speed]: c7794709e69683b3 == %016" PRIx64 "\n", + (uint64_t)crc64(0, li, sizeof(li))); return 0; } + +#endif + +#ifdef REDIS_TEST_MAIN +int main(int argc, char *argv[]) { + return crc64Test(argc, argv); +} + #endif diff --git a/redis.submodule/src/crc64.h b/redis.submodule/src/crc64.h index c9fca51..60c4234 100644 --- a/redis.submodule/src/crc64.h +++ b/redis.submodule/src/crc64.h @@ -3,6 +3,7 @@ #include +void crc64_init(void); uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); #ifdef REDIS_TEST diff --git a/redis.submodule/src/crcspeed.c b/redis.submodule/src/crcspeed.c new file mode 100644 index 0000000..d2d97a8 --- /dev/null +++ b/redis.submodule/src/crcspeed.c @@ -0,0 +1,281 @@ +/* + * Copyright (C) 2013 Mark Adler + * Originally by: crc64.c Version 1.4 16 Dec 2013 Mark Adler + * Modifications by Matt Stancliff : + * - removed CRC64-specific behavior + * - added generation of lookup tables by parameters + * - removed inversion of CRC input/result + * - removed automatic initialization in favor of explicit initialization + + This software is provided 'as-is', without any express or implied + warranty. In no event will the author be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Mark Adler + madler@alumni.caltech.edu + */ + +#include "crcspeed.h" + +/* Fill in a CRC constants table. */ +void crcspeed64little_init(crcfn64 crcfn, uint64_t table[8][256]) { + uint64_t crc; + + /* generate CRCs for all single byte sequences */ + for (int n = 0; n < 256; n++) { + table[0][n] = crcfn(0, &n, 1); + } + + /* generate nested CRC table for future slice-by-8 lookup */ + for (int n = 0; n < 256; n++) { + crc = table[0][n]; + for (int k = 1; k < 8; k++) { + crc = table[0][crc & 0xff] ^ (crc >> 8); + table[k][n] = crc; + } + } +} + +void crcspeed16little_init(crcfn16 crcfn, uint16_t table[8][256]) { + uint16_t crc; + + /* generate CRCs for all single byte sequences */ + for (int n = 0; n < 256; n++) { + table[0][n] = crcfn(0, &n, 1); + } + + /* generate nested CRC table for future slice-by-8 lookup */ + for (int n = 0; n < 256; n++) { + crc = table[0][n]; + for (int k = 1; k < 8; k++) { + crc = table[0][(crc >> 8) & 0xff] ^ (crc << 8); + table[k][n] = crc; + } + } +} + +/* Reverse the bytes in a 64-bit word. */ +static inline uint64_t rev8(uint64_t a) { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap64(a); +#else + uint64_t m; + + m = UINT64_C(0xff00ff00ff00ff); + a = ((a >> 8) & m) | (a & m) << 8; + m = UINT64_C(0xffff0000ffff); + a = ((a >> 16) & m) | (a & m) << 16; + return a >> 32 | a << 32; +#endif +} + +/* This function is called once to initialize the CRC table for use on a + big-endian architecture. */ +void crcspeed64big_init(crcfn64 fn, uint64_t big_table[8][256]) { + /* Create the little endian table then reverse all the entires. */ + crcspeed64little_init(fn, big_table); + for (int k = 0; k < 8; k++) { + for (int n = 0; n < 256; n++) { + big_table[k][n] = rev8(big_table[k][n]); + } + } +} + +void crcspeed16big_init(crcfn16 fn, uint16_t big_table[8][256]) { + /* Create the little endian table then reverse all the entires. */ + crcspeed16little_init(fn, big_table); + for (int k = 0; k < 8; k++) { + for (int n = 0; n < 256; n++) { + big_table[k][n] = rev8(big_table[k][n]); + } + } +} + +/* Calculate a non-inverted CRC multiple bytes at a time on a little-endian + * architecture. If you need inverted CRC, invert *before* calling and invert + * *after* calling. + * 64 bit crc = process 8 bytes at once; + */ +uint64_t crcspeed64little(uint64_t little_table[8][256], uint64_t crc, + void *buf, size_t len) { + unsigned char *next = buf; + + /* process individual bytes until we reach an 8-byte aligned pointer */ + while (len && ((uintptr_t)next & 7) != 0) { + crc = little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8); + len--; + } + + /* fast middle processing, 8 bytes (aligned!) per loop */ + while (len >= 8) { + crc ^= *(uint64_t *)next; + crc = little_table[7][crc & 0xff] ^ + little_table[6][(crc >> 8) & 0xff] ^ + little_table[5][(crc >> 16) & 0xff] ^ + little_table[4][(crc >> 24) & 0xff] ^ + little_table[3][(crc >> 32) & 0xff] ^ + little_table[2][(crc >> 40) & 0xff] ^ + little_table[1][(crc >> 48) & 0xff] ^ + little_table[0][crc >> 56]; + next += 8; + len -= 8; + } + + /* process remaining bytes (can't be larger than 8) */ + while (len) { + crc = little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8); + len--; + } + + return crc; +} + +uint16_t crcspeed16little(uint16_t little_table[8][256], uint16_t crc, + void *buf, size_t len) { + unsigned char *next = buf; + + /* process individual bytes until we reach an 8-byte aligned pointer */ + while (len && ((uintptr_t)next & 7) != 0) { + crc = little_table[0][((crc >> 8) ^ *next++) & 0xff] ^ (crc << 8); + len--; + } + + /* fast middle processing, 8 bytes (aligned!) per loop */ + while (len >= 8) { + uint64_t n = *(uint64_t *)next; + crc = little_table[7][(n & 0xff) ^ ((crc >> 8) & 0xff)] ^ + little_table[6][((n >> 8) & 0xff) ^ (crc & 0xff)] ^ + little_table[5][(n >> 16) & 0xff] ^ + little_table[4][(n >> 24) & 0xff] ^ + little_table[3][(n >> 32) & 0xff] ^ + little_table[2][(n >> 40) & 0xff] ^ + little_table[1][(n >> 48) & 0xff] ^ + little_table[0][n >> 56]; + next += 8; + len -= 8; + } + + /* process remaining bytes (can't be larger than 8) */ + while (len) { + crc = little_table[0][((crc >> 8) ^ *next++) & 0xff] ^ (crc << 8); + len--; + } + + return crc; +} + +/* Calculate a non-inverted CRC eight bytes at a time on a big-endian + * architecture. + */ +uint64_t crcspeed64big(uint64_t big_table[8][256], uint64_t crc, void *buf, + size_t len) { + unsigned char *next = buf; + + crc = rev8(crc); + while (len && ((uintptr_t)next & 7) != 0) { + crc = big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8); + len--; + } + + while (len >= 8) { + crc ^= *(uint64_t *)next; + crc = big_table[0][crc & 0xff] ^ + big_table[1][(crc >> 8) & 0xff] ^ + big_table[2][(crc >> 16) & 0xff] ^ + big_table[3][(crc >> 24) & 0xff] ^ + big_table[4][(crc >> 32) & 0xff] ^ + big_table[5][(crc >> 40) & 0xff] ^ + big_table[6][(crc >> 48) & 0xff] ^ + big_table[7][crc >> 56]; + next += 8; + len -= 8; + } + + while (len) { + crc = big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8); + len--; + } + + return rev8(crc); +} + +/* WARNING: Completely untested on big endian architecture. Possibly broken. */ +uint16_t crcspeed16big(uint16_t big_table[8][256], uint16_t crc_in, void *buf, + size_t len) { + unsigned char *next = buf; + uint64_t crc = crc_in; + + crc = rev8(crc); + while (len && ((uintptr_t)next & 7) != 0) { + crc = big_table[0][((crc >> (56 - 8)) ^ *next++) & 0xff] ^ (crc >> 8); + len--; + } + + while (len >= 8) { + uint64_t n = *(uint64_t *)next; + crc = big_table[0][(n & 0xff) ^ ((crc >> (56 - 8)) & 0xff)] ^ + big_table[1][((n >> 8) & 0xff) ^ (crc & 0xff)] ^ + big_table[2][(n >> 16) & 0xff] ^ + big_table[3][(n >> 24) & 0xff] ^ + big_table[4][(n >> 32) & 0xff] ^ + big_table[5][(n >> 40) & 0xff] ^ + big_table[6][(n >> 48) & 0xff] ^ + big_table[7][n >> 56]; + next += 8; + len -= 8; + } + + while (len) { + crc = big_table[0][((crc >> (56 - 8)) ^ *next++) & 0xff] ^ (crc >> 8); + len--; + } + + return rev8(crc); +} + +/* Return the CRC of buf[0..len-1] with initial crc, processing eight bytes + at a time using passed-in lookup table. + This selects one of two routines depending on the endianess of + the architecture. */ +uint64_t crcspeed64native(uint64_t table[8][256], uint64_t crc, void *buf, + size_t len) { + uint64_t n = 1; + + return *(char *)&n ? crcspeed64little(table, crc, buf, len) + : crcspeed64big(table, crc, buf, len); +} + +uint16_t crcspeed16native(uint16_t table[8][256], uint16_t crc, void *buf, + size_t len) { + uint64_t n = 1; + + return *(char *)&n ? crcspeed16little(table, crc, buf, len) + : crcspeed16big(table, crc, buf, len); +} + +/* Initialize CRC lookup table in architecture-dependent manner. */ +void crcspeed64native_init(crcfn64 fn, uint64_t table[8][256]) { + uint64_t n = 1; + + *(char *)&n ? crcspeed64little_init(fn, table) + : crcspeed64big_init(fn, table); +} + +void crcspeed16native_init(crcfn16 fn, uint16_t table[8][256]) { + uint64_t n = 1; + + *(char *)&n ? crcspeed16little_init(fn, table) + : crcspeed16big_init(fn, table); +} diff --git a/redis.submodule/src/crcspeed.h b/redis.submodule/src/crcspeed.h new file mode 100644 index 0000000..d7ee95e --- /dev/null +++ b/redis.submodule/src/crcspeed.h @@ -0,0 +1,60 @@ +/* Copyright (c) 2014, Matt Stancliff + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. */ + +#ifndef CRCSPEED_H +#define CRCSPEED_H + +#include +#include + +typedef uint64_t (*crcfn64)(uint64_t, const void *, const uint64_t); +typedef uint16_t (*crcfn16)(uint16_t, const void *, const uint64_t); + +/* CRC-64 */ +void crcspeed64little_init(crcfn64 fn, uint64_t table[8][256]); +void crcspeed64big_init(crcfn64 fn, uint64_t table[8][256]); +void crcspeed64native_init(crcfn64 fn, uint64_t table[8][256]); + +uint64_t crcspeed64little(uint64_t table[8][256], uint64_t crc, void *buf, + size_t len); +uint64_t crcspeed64big(uint64_t table[8][256], uint64_t crc, void *buf, + size_t len); +uint64_t crcspeed64native(uint64_t table[8][256], uint64_t crc, void *buf, + size_t len); + +/* CRC-16 */ +void crcspeed16little_init(crcfn16 fn, uint16_t table[8][256]); +void crcspeed16big_init(crcfn16 fn, uint16_t table[8][256]); +void crcspeed16native_init(crcfn16 fn, uint16_t table[8][256]); + +uint16_t crcspeed16little(uint16_t table[8][256], uint16_t crc, void *buf, + size_t len); +uint16_t crcspeed16big(uint16_t table[8][256], uint16_t crc, void *buf, + size_t len); +uint16_t crcspeed16native(uint16_t table[8][256], uint16_t crc, void *buf, + size_t len); +#endif diff --git a/redis.submodule/src/db.c b/redis.submodule/src/db.c index 62c8aa1..dc4a0b6 100644 --- a/redis.submodule/src/db.c +++ b/redis.submodule/src/db.c @@ -60,10 +60,7 @@ robj *lookupKey(redisDb *db, robj *key, int flags) { /* Update the access time for the ageing algorithm. * Don't do it if we have a saving child, as this will trigger * a copy on write madness. */ - if (server.rdb_child_pid == -1 && - server.aof_child_pid == -1 && - !(flags & LOOKUP_NOTOUCH)) - { + if (!hasActiveChildProcess() && !(flags & LOOKUP_NOTOUCH)){ if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { updateLFU(val); } else { @@ -83,6 +80,7 @@ robj *lookupKey(redisDb *db, robj *key, int flags) { * 1. A key gets expired if it reached it's TTL. * 2. The key last access time is updated. * 3. The global keys hits/misses stats are updated (reported in INFO). + * 4. If keyspace notifications are enabled, a "keymiss" notification is fired. * * This API should not be used when we write to the key after obtaining * the object linked to the key, but only for read only operations. @@ -106,6 +104,7 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) { * to return NULL ASAP. */ if (server.masterhost == NULL) { server.stat_keyspace_misses++; + notifyKeyspaceEvent(NOTIFY_KEY_MISS, "keymiss", key, db->id); return NULL; } @@ -127,12 +126,15 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) { server.current_client->cmd->flags & CMD_READONLY) { server.stat_keyspace_misses++; + notifyKeyspaceEvent(NOTIFY_KEY_MISS, "keymiss", key, db->id); return NULL; } } val = lookupKey(db,key,flags); - if (val == NULL) + if (val == NULL) { server.stat_keyspace_misses++; + notifyKeyspaceEvent(NOTIFY_KEY_MISS, "keymiss", key, db->id); + } else server.stat_keyspace_hits++; return val; @@ -149,9 +151,13 @@ robj *lookupKeyRead(redisDb *db, robj *key) { * * Returns the linked value object if the key exists or NULL if the key * does not exist in the specified DB. */ -robj *lookupKeyWrite(redisDb *db, robj *key) { +robj *lookupKeyWriteWithFlags(redisDb *db, robj *key, int flags) { expireIfNeeded(db,key); - return lookupKey(db,key,LOOKUP_NONE); + return lookupKey(db,key,flags); +} + +robj *lookupKeyWrite(redisDb *db, robj *key) { + return lookupKeyWriteWithFlags(db, key, LOOKUP_NONE); } robj *lookupKeyReadOrReply(client *c, robj *key, robj *reply) { @@ -176,9 +182,28 @@ void dbAdd(redisDb *db, robj *key, robj *val) { serverAssertWithInfo(NULL,key,retval == DICT_OK); if (val->type == OBJ_LIST || - val->type == OBJ_ZSET) + val->type == OBJ_ZSET || + val->type == OBJ_STREAM) signalKeyAsReady(db, key); + if (server.cluster_enabled) slotToKeyAdd(key->ptr); +} + +/* This is a special version of dbAdd() that is used only when loading + * keys from the RDB file: the key is passed as an SDS string that is + * retained by the function (and not freed by the caller). + * + * Moreover this function will not abort if the key is already busy, to + * give more control to the caller, nor will signal the key as ready + * since it is not useful in this context. + * + * The function returns 1 if the key was added to the database, taking + * ownership of the SDS string, otherwise 0 is returned, and is up to the + * caller to free the SDS string. */ +int dbAddRDBLoad(redisDb *db, sds key, robj *val) { + int retval = dictAdd(db->dict, key, val); + if (retval != DICT_OK) return 0; if (server.cluster_enabled) slotToKeyAdd(key); + return 1; } /* Overwrite an existing key with a new value. Incrementing the reference @@ -210,20 +235,30 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) { * * 1) The ref count of the value object is incremented. * 2) clients WATCHing for the destination key notified. - * 3) The expire time of the key is reset (the key is made persistent). + * 3) The expire time of the key is reset (the key is made persistent), + * unless 'keepttl' is true. * - * All the new keys in the database should be created via this interface. */ -void setKey(redisDb *db, robj *key, robj *val) { + * All the new keys in the database should be created via this interface. + * The client 'c' argument may be set to NULL if the operation is performed + * in a context where there is no clear client performing the operation. */ +void genericSetKey(client *c, redisDb *db, robj *key, robj *val, int keepttl, int signal) { if (lookupKeyWrite(db,key) == NULL) { dbAdd(db,key,val); } else { dbOverwrite(db,key,val); } incrRefCount(val); - removeExpire(db,key); - signalModifiedKey(db,key); + if (!keepttl) removeExpire(db,key); + if (signal) signalModifiedKey(c,db,key); +} + +/* Common case for genericSetKey() where the TTL is not retained. */ +void setKey(client *c, redisDb *db, robj *key, robj *val) { + genericSetKey(c,db,key,val,0,1); } +/* Return true if the specified key exists in the specified database. + * LRU/LFU info is not updated in any way. */ int dbExists(redisDb *db, robj *key) { return dictFind(db->dict,key->ptr) != NULL; } @@ -241,7 +276,7 @@ robj *dbRandomKey(redisDb *db) { sds key; robj *keyobj; - de = dictGetRandomKey(db->dict); + de = dictGetFairRandomKey(db->dict); if (de == NULL) return NULL; key = dictGetKey(de); @@ -273,7 +308,7 @@ int dbSyncDelete(redisDb *db, robj *key) { * the key, because it is shared with the main dictionary. */ if (dictSize(db->expires) > 0) dictDelete(db->expires,key->ptr); if (dictDelete(db->dict,key->ptr) == DICT_OK) { - if (server.cluster_enabled) slotToKeyDel(key); + if (server.cluster_enabled) slotToKeyDel(key->ptr); return 1; } else { return 0; @@ -333,14 +368,19 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) { * DB number if we want to flush only a single Redis database number. * * Flags are be EMPTYDB_NO_FLAGS if no special flags are specified or - * EMPTYDB_ASYNC if we want the memory to be freed in a different thread + * 1. EMPTYDB_ASYNC if we want the memory to be freed in a different thread. + * 2. EMPTYDB_BACKUP if we want to empty the backup dictionaries created by + * disklessLoadMakeBackups. In that case we only free memory and avoid + * firing module events. * and the function to return ASAP. * * On success the fuction returns the number of keys removed from the * database(s). Otherwise -1 is returned in the specific case the * DB number is out of range, and errno is set to EINVAL. */ -long long emptyDb(int dbnum, int flags, void(callback)(void*)) { +long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(void*)) { int async = (flags & EMPTYDB_ASYNC); + int backup = (flags & EMPTYDB_BACKUP); /* Just free the memory, nothing else */ + RedisModuleFlushInfoV1 fi = {REDISMODULE_FLUSHINFO_VERSION,!async,dbnum}; long long removed = 0; if (dbnum < -1 || dbnum >= server.dbnum) { @@ -348,6 +388,19 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { return -1; } + /* Pre-flush actions */ + if (!backup) { + /* Fire the flushdb modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, + REDISMODULE_SUBEVENT_FLUSHDB_START, + &fi); + + /* Make sure the WATCHed keys are affected by the FLUSH* commands. + * Note that we need to call the function while the keys are still + * there. */ + signalFlushedDb(dbnum); + } + int startdb, enddb; if (dbnum == -1) { startdb = 0; @@ -357,25 +410,40 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { } for (int j = startdb; j <= enddb; j++) { - removed += dictSize(server.db[j].dict); + removed += dictSize(dbarray[j].dict); if (async) { - emptyDbAsync(&server.db[j]); + emptyDbAsync(&dbarray[j]); } else { - dictEmpty(server.db[j].dict,callback); - dictEmpty(server.db[j].expires,callback); + dictEmpty(dbarray[j].dict,callback); + dictEmpty(dbarray[j].expires,callback); } } - if (server.cluster_enabled) { - if (async) { - slotToKeyFlushAsync(); - } else { - slotToKeyFlush(); + + /* Post-flush actions */ + if (!backup) { + if (server.cluster_enabled) { + if (async) { + slotToKeyFlushAsync(); + } else { + slotToKeyFlush(); + } } + if (dbnum == -1) flushSlaveKeysWithExpireList(); + + /* Also fire the end event. Note that this event will fire almost + * immediately after the start event if the flush is asynchronous. */ + moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, + REDISMODULE_SUBEVENT_FLUSHDB_END, + &fi); } - if (dbnum == -1) flushSlaveKeysWithExpireList(); + return removed; } +long long emptyDb(int dbnum, int flags, void(callback)(void*)) { + return emptyDbGeneric(server.db, dbnum, flags, callback); +} + int selectDb(client *c, int id) { if (id < 0 || id >= server.dbnum) return C_ERR; @@ -383,6 +451,15 @@ int selectDb(client *c, int id) { return C_OK; } +long long dbTotalServerKeyCount() { + long long total = 0; + int j; + for (j = 0; j < server.dbnum; j++) { + total += dictSize(server.db[j].dict); + } + return total; +} + /*----------------------------------------------------------------------------- * Hooks for key space changes. * @@ -392,12 +469,16 @@ int selectDb(client *c, int id) { * Every time a DB is flushed the function signalFlushDb() is called. *----------------------------------------------------------------------------*/ -void signalModifiedKey(redisDb *db, robj *key) { +/* Note that the 'c' argument may be NULL if the key was modified out of + * a context of a client. */ +void signalModifiedKey(client *c, redisDb *db, robj *key) { touchWatchedKey(db,key); + trackingInvalidateKey(c,key); } void signalFlushedDb(int dbid) { touchWatchedKeysOnFlush(dbid); + trackingInvalidateKeysOnFlush(dbid); } /*----------------------------------------------------------------------------- @@ -426,6 +507,29 @@ int getFlushCommandFlags(client *c, int *flags) { return C_OK; } +/* Flushes the whole server data set. */ +void flushAllDataAndResetRDB(int flags) { + server.dirty += emptyDb(-1,flags,NULL); + if (server.rdb_child_pid != -1) killRDBChild(); + if (server.saveparamslen > 0) { + /* Normally rdbSave() will reset dirty, but we don't want this here + * as otherwise FLUSHALL will not be replicated nor put into the AOF. */ + int saved_dirty = server.dirty; + rdbSaveInfo rsi, *rsiptr; + rsiptr = rdbPopulateSaveInfo(&rsi); + rdbSave(server.rdb_filename,rsiptr); + server.dirty = saved_dirty; + } + server.dirty++; +#if defined(USE_JEMALLOC) + /* jemalloc 5 doesn't release pages back to the OS when there's no traffic. + * for large databases, flushdb blocks for long anyway, so a bit more won't + * harm and this way the flush and purge will be synchroneus. */ + if (!(flags & EMPTYDB_ASYNC)) + jemalloc_purge(); +#endif +} + /* FLUSHDB [ASYNC] * * Flushes the currently SELECTed Redis DB. */ @@ -433,9 +537,15 @@ void flushdbCommand(client *c) { int flags; if (getFlushCommandFlags(c,&flags) == C_ERR) return; - signalFlushedDb(c->db->id); server.dirty += emptyDb(c->db->id,flags,NULL); addReply(c,shared.ok); +#if defined(USE_JEMALLOC) + /* jemalloc 5 doesn't release pages back to the OS when there's no traffic. + * for large databases, flushdb blocks for long anyway, so a bit more won't + * harm and this way the flush and purge will be synchroneus. */ + if (!(flags & EMPTYDB_ASYNC)) + jemalloc_purge(); +#endif } /* FLUSHALL [ASYNC] @@ -443,25 +553,9 @@ void flushdbCommand(client *c) { * Flushes the whole server data set. */ void flushallCommand(client *c) { int flags; - if (getFlushCommandFlags(c,&flags) == C_ERR) return; - signalFlushedDb(-1); - server.dirty += emptyDb(-1,flags,NULL); + flushAllDataAndResetRDB(flags); addReply(c,shared.ok); - if (server.rdb_child_pid != -1) { - kill(server.rdb_child_pid,SIGUSR1); - rdbRemoveTempFile(server.rdb_child_pid); - } - if (server.saveparamslen > 0) { - /* Normally rdbSave() will reset dirty, but we don't want this here - * as otherwise FLUSHALL will not be replicated nor put into the AOF. */ - int saved_dirty = server.dirty; - rdbSaveInfo rsi, *rsiptr; - rsiptr = rdbPopulateSaveInfo(&rsi); - rdbSave(server.rdb_filename,rsiptr); - server.dirty = saved_dirty; - } - server.dirty++; } /* This command implements DEL and LAZYDEL. */ @@ -473,7 +567,7 @@ void delGenericCommand(client *c, int lazy) { int deleted = lazy ? dbAsyncDelete(c->db,c->argv[j]) : dbSyncDelete(c->db,c->argv[j]); if (deleted) { - signalModifiedKey(c->db,c->argv[j]); + signalModifiedKey(c,c->db,c->argv[j]); notifyKeyspaceEvent(NOTIFY_GENERIC, "del",c->argv[j],c->db->id); server.dirty++; @@ -484,7 +578,7 @@ void delGenericCommand(client *c, int lazy) { } void delCommand(client *c) { - delGenericCommand(c,0); + delGenericCommand(c,server.lazyfree_lazy_user_del); } void unlinkCommand(client *c) { @@ -525,7 +619,7 @@ void randomkeyCommand(client *c) { robj *key; if ((key = dbRandomKey(c->db)) == NULL) { - addReply(c,shared.nullbulk); + addReplyNull(c); return; } @@ -539,10 +633,10 @@ void keysCommand(client *c) { sds pattern = c->argv[1]->ptr; int plen = sdslen(pattern), allkeys; unsigned long numkeys = 0; - void *replylen = addDeferredMultiBulkLength(c); + void *replylen = addReplyDeferredLen(c); di = dictGetSafeIterator(c->db->dict); - allkeys = (pattern[0] == '*' && pattern[1] == '\0'); + allkeys = (pattern[0] == '*' && plen == 1); while((de = dictNext(di)) != NULL) { sds key = dictGetKey(de); robj *keyobj; @@ -557,7 +651,7 @@ void keysCommand(client *c) { } } dictReleaseIterator(di); - setDeferredMultiBulkLength(c,replylen,numkeys); + setDeferredArrayLen(c,replylen,numkeys); } /* This callback is used by scanGenericCommand in order to collect elements @@ -611,7 +705,7 @@ int parseScanCursorOrReply(client *c, robj *o, unsigned long *cursor) { } /* This command implements SCAN, HSCAN and SSCAN commands. - * If object 'o' is passed, then it must be a Hash or Set object, otherwise + * If object 'o' is passed, then it must be a Hash, Set or Zset object, otherwise * if 'o' is NULL the command will operate on the dictionary associated with * the current database. * @@ -627,6 +721,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { listNode *node, *nextnode; long count = 10; sds pat = NULL; + sds typename = NULL; int patlen = 0, use_pattern = 0; dict *ht; @@ -663,6 +758,10 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { use_pattern = !(pat[0] == '*' && patlen == 1); i += 2; + } else if (!strcasecmp(c->argv[i]->ptr, "type") && o == NULL && j >= 2) { + /* SCAN for a particular type only applies to the db dict */ + typename = c->argv[i+1]->ptr; + i+= 2; } else { addReply(c,shared.syntaxerr); goto cleanup; @@ -757,6 +856,13 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { } } + /* Filter an element if it isn't the type we want. */ + if (!filter && o == NULL && typename){ + robj* typecheck = lookupKeyReadWithFlags(c->db, kobj, LOOKUP_NOTOUCH); + char* type = getObjectTypeName(typecheck); + if (strcasecmp((char*) typename, type)) filter = 1; + } + /* Filter element if it is an expired key. */ if (!filter && o == NULL && expireIfNeeded(c->db, kobj)) filter = 1; @@ -782,10 +888,10 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { } /* Step 4: Reply to the client. */ - addReplyMultiBulkLen(c, 2); + addReplyArrayLen(c, 2); addReplyBulkLongLong(c,cursor); - addReplyMultiBulkLen(c, listLength(keys)); + addReplyArrayLen(c, listLength(keys)); while ((node = listFirst(keys)) != NULL) { robj *kobj = listNodeValue(node); addReplyBulk(c, kobj); @@ -813,11 +919,8 @@ void lastsaveCommand(client *c) { addReplyLongLong(c,server.lastsave); } -void typeCommand(client *c) { - robj *o; - char *type; - - o = lookupKeyReadWithFlags(c->db,c->argv[1],LOOKUP_NOTOUCH); +char* getObjectTypeName(robj *o) { + char* type; if (o == NULL) { type = "none"; } else { @@ -835,7 +938,13 @@ void typeCommand(client *c) { default: type = "unknown"; break; } } - addReplyStatus(c,type); + return type; +} + +void typeCommand(client *c) { + robj *o; + o = lookupKeyReadWithFlags(c->db,c->argv[1],LOOKUP_NOTOUCH); + addReplyStatus(c, getObjectTypeName(o)); } void shutdownCommand(client *c) { @@ -898,8 +1007,8 @@ void renameGenericCommand(client *c, int nx) { dbAdd(c->db,c->argv[2],o); if (expire != -1) setExpire(c,c->db,c->argv[2],expire); dbDelete(c->db,c->argv[1]); - signalModifiedKey(c->db,c->argv[1]); - signalModifiedKey(c->db,c->argv[2]); + signalModifiedKey(c,c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[2]); notifyKeyspaceEvent(NOTIFY_GENERIC,"rename_from", c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_GENERIC,"rename_to", @@ -967,6 +1076,13 @@ void moveCommand(client *c) { /* OK! key moved, free the entry in the source DB */ dbDelete(src,c->argv[1]); + signalModifiedKey(c,src,c->argv[1]); + signalModifiedKey(c,dst,c->argv[1]); + notifyKeyspaceEvent(NOTIFY_GENERIC, + "move_from",c->argv[1],src->id); + notifyKeyspaceEvent(NOTIFY_GENERIC, + "move_to",c->argv[1],dst->id); + server.dirty++; addReply(c,shared.cone); } @@ -997,7 +1113,7 @@ void scanDatabaseForReadyLists(redisDb *db) { * * Returns C_ERR if at least one of the DB ids are out of range, otherwise * C_OK is returned. */ -int dbSwapDatabases(int id1, int id2) { +int dbSwapDatabases(long id1, long id2) { if (id1 < 0 || id1 >= server.dbnum || id2 < 0 || id2 >= server.dbnum) return C_ERR; if (id1 == id2) return C_OK; @@ -1010,10 +1126,12 @@ int dbSwapDatabases(int id1, int id2) { db1->dict = db2->dict; db1->expires = db2->expires; db1->avg_ttl = db2->avg_ttl; + db1->expires_cursor = db2->expires_cursor; db2->dict = aux.dict; db2->expires = aux.expires; db2->avg_ttl = aux.avg_ttl; + db2->expires_cursor = aux.expires_cursor; /* Now we need to handle clients blocked on lists: as an effect * of swapping the two DBs, a client that was waiting for list @@ -1129,6 +1247,7 @@ void propagateExpire(redisDb *db, robj *key, int lazy) { /* Check if the key is expired. */ int keyIsExpired(redisDb *db, robj *key) { mstime_t when = getExpire(db,key); + mstime_t now; if (when < 0) return 0; /* No expire for this key */ @@ -1140,8 +1259,26 @@ int keyIsExpired(redisDb *db, robj *key) { * only the first time it is accessed and not in the middle of the * script execution, making propagation to slaves / AOF consistent. * See issue #1525 on Github for more information. */ - mstime_t now = server.lua_caller ? server.lua_time_start : mstime(); + if (server.lua_caller) { + now = server.lua_time_start; + } + /* If we are in the middle of a command execution, we still want to use + * a reference time that does not change: in that case we just use the + * cached time, that we update before each call in the call() function. + * This way we avoid that commands such as RPOPLPUSH or similar, that + * may re-open the same key multiple times, can invalidate an already + * open object in a next call, if the next call will see the key expired, + * while the first did not. */ + else if (server.fixed_time_expire > 0) { + now = server.mstime; + } + /* For the other cases, we want to use the most fresh time we have. */ + else { + now = mstime(); + } + /* The key expired if the current (virtual or real) time is greater + * than the expire time of the key. */ return now > when; } @@ -1182,13 +1319,17 @@ int expireIfNeeded(redisDb *db, robj *key) { propagateExpire(db,key,server.lazyfree_lazy_expire); notifyKeyspaceEvent(NOTIFY_EXPIRED, "expired",key,db->id); - return server.lazyfree_lazy_expire ? dbAsyncDelete(db,key) : - dbSyncDelete(db,key); + int retval = server.lazyfree_lazy_expire ? dbAsyncDelete(db,key) : + dbSyncDelete(db,key); + if (retval) signalModifiedKey(NULL,db,key); + return retval; } /* ----------------------------------------------------------------------------- * API to get key arguments from commands * ---------------------------------------------------------------------------*/ +#define MAX_KEYS_BUFFER 256 +static int getKeysTempBuffer[MAX_KEYS_BUFFER]; /* The base case is to use the keys position as given in the command table * (firstkey, lastkey, step). */ @@ -1203,7 +1344,12 @@ int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, in last = cmd->lastkey; if (last < 0) last = argc+last; - keys = zmalloc(sizeof(int)*((last - cmd->firstkey)+1)); + + int count = ((last - cmd->firstkey)+1); + keys = getKeysTempBuffer; + if (count > MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int)*count); + for (j = cmd->firstkey; j <= last; j += cmd->keystep) { if (j >= argc) { /* Modules commands, and standard commands with a not fixed number @@ -1213,7 +1359,7 @@ int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, in * return no keys and expect the command implementation to report * an arity or syntax error. */ if (cmd->flags & CMD_MODULE || cmd->arity < 0) { - zfree(keys); + getKeysFreeResult(keys); *numkeys = 0; return NULL; } else { @@ -1249,7 +1395,8 @@ int *getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, int *nu /* Free the result of getKeysFromCommand. */ void getKeysFreeResult(int *result) { - zfree(result); + if (result != getKeysTempBuffer) + zfree(result); } /* Helper function to extract keys from following commands: @@ -1270,7 +1417,9 @@ int *zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *nu /* Keys in z{union,inter}store come from two places: * argv[1] = storage key, * argv[3...n] = keys to intersect */ - keys = zmalloc(sizeof(int)*(num+1)); + keys = getKeysTempBuffer; + if (num+1>MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int)*(num+1)); /* Add all key positions for argv[3...n] to keys[] */ for (i = 0; i < num; i++) keys[i] = 3+i; @@ -1296,7 +1445,10 @@ int *evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) return NULL; } - keys = zmalloc(sizeof(int)*num); + keys = getKeysTempBuffer; + if (num>MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int)*num); + *numkeys = num; /* Add all key positions for argv[3...n] to keys[] */ @@ -1317,7 +1469,7 @@ int *sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) UNUSED(cmd); num = 0; - keys = zmalloc(sizeof(int)*2); /* Alloc 2 places for the worst case. */ + keys = getKeysTempBuffer; /* Alloc 2 places for the worst case. */ keys[num++] = 1; /* is always present. */ @@ -1375,7 +1527,10 @@ int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkey } } - keys = zmalloc(sizeof(int)*num); + keys = getKeysTempBuffer; + if (num>MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int)*num); + for (i = 0; i < num; i++) keys[i] = first+i; *numkeys = num; return keys; @@ -1408,7 +1563,9 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk * argv[1] = key, * argv[5...n] = stored key if present */ - keys = zmalloc(sizeof(int) * num); + keys = getKeysTempBuffer; + if (num>MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int) * num); /* Add all key positions to keys[] */ keys[0] = 1; @@ -1419,6 +1576,48 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk return keys; } +/* LCS ... [KEYS ] ... */ +int *lcsGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) +{ + int i; + int *keys = getKeysTempBuffer; + UNUSED(cmd); + + /* We need to parse the options of the command in order to check for the + * "KEYS" argument before the "STRINGS" argument. */ + for (i = 1; i < argc; i++) { + char *arg = argv[i]->ptr; + int moreargs = (argc-1) - i; + + if (!strcasecmp(arg, "strings")) { + break; + } else if (!strcasecmp(arg, "keys") && moreargs >= 2) { + keys[0] = i+1; + keys[1] = i+2; + *numkeys = 2; + return keys; + } + } + *numkeys = 0; + return keys; +} + +/* Helper function to extract keys from memory command. + * MEMORY USAGE */ +int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) { + int *keys; + UNUSED(cmd); + + if (argc >= 3 && !strcasecmp(argv[1]->ptr,"usage")) { + keys = getKeysTempBuffer; + keys[0] = 2; + *numkeys = 1; + return keys; + } + *numkeys = 0; + return NULL; +} + /* XREAD [BLOCK ] [COUNT ] [GROUP ] * STREAMS key_1 key_2 ... key_N ID_1 ID_2 ... ID_N */ int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) { @@ -1457,7 +1656,10 @@ int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) num /= 2; /* We have half the keys as there are arguments because there are also the IDs, one per key. */ - keys = zmalloc(sizeof(int) * num); + keys = getKeysTempBuffer; + if (num>MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int) * num); + for (i = streams_pos+1; i < argc-num; i++) keys[i-streams_pos-1] = i; *numkeys = num; return keys; @@ -1467,17 +1669,17 @@ int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) * a fast way a key that belongs to a specified hash slot. This is useful * while rehashing the cluster and in other conditions when we need to * understand if we have keys for a given hash slot. */ -void slotToKeyUpdateKey(robj *key, int add) { - unsigned int hashslot = keyHashSlot(key->ptr,sdslen(key->ptr)); +void slotToKeyUpdateKey(sds key, int add) { + size_t keylen = sdslen(key); + unsigned int hashslot = keyHashSlot(key,keylen); unsigned char buf[64]; unsigned char *indexed = buf; - size_t keylen = sdslen(key->ptr); server.cluster->slots_keys_count[hashslot] += add ? 1 : -1; if (keylen+2 > 64) indexed = zmalloc(keylen+2); indexed[0] = (hashslot >> 8) & 0xff; indexed[1] = hashslot & 0xff; - memcpy(indexed+2,key->ptr,keylen); + memcpy(indexed+2,key,keylen); if (add) { raxInsert(server.cluster->slots_to_keys,indexed,keylen+2,NULL,NULL); } else { @@ -1486,11 +1688,11 @@ void slotToKeyUpdateKey(robj *key, int add) { if (indexed != buf) zfree(indexed); } -void slotToKeyAdd(robj *key) { +void slotToKeyAdd(sds key) { slotToKeyUpdateKey(key,1); } -void slotToKeyDel(robj *key) { +void slotToKeyDel(sds key) { slotToKeyUpdateKey(key,0); } diff --git a/redis.submodule/src/debug.c b/redis.submodule/src/debug.c index 1ec7c49..d79226b 100644 --- a/redis.submodule/src/debug.c +++ b/redis.submodule/src/debug.c @@ -297,6 +297,76 @@ void computeDatasetDigest(unsigned char *final) { } } +#ifdef USE_JEMALLOC +void mallctl_int(client *c, robj **argv, int argc) { + int ret; + /* start with the biggest size (int64), and if that fails, try smaller sizes (int32, bool) */ + int64_t old = 0, val; + if (argc > 1) { + long long ll; + if (getLongLongFromObjectOrReply(c, argv[1], &ll, NULL) != C_OK) + return; + val = ll; + } + size_t sz = sizeof(old); + while (sz > 0) { + if ((ret=je_mallctl(argv[0]->ptr, &old, &sz, argc > 1? &val: NULL, argc > 1?sz: 0))) { + if (ret == EPERM && argc > 1) { + /* if this option is write only, try just writing to it. */ + if (!(ret=je_mallctl(argv[0]->ptr, NULL, 0, &val, sz))) { + addReply(c, shared.ok); + return; + } + } + if (ret==EINVAL) { + /* size might be wrong, try a smaller one */ + sz /= 2; +#if BYTE_ORDER == BIG_ENDIAN + val <<= 8*sz; +#endif + continue; + } + addReplyErrorFormat(c,"%s", strerror(ret)); + return; + } else { +#if BYTE_ORDER == BIG_ENDIAN + old >>= 64 - 8*sz; +#endif + addReplyLongLong(c, old); + return; + } + } + addReplyErrorFormat(c,"%s", strerror(EINVAL)); +} + +void mallctl_string(client *c, robj **argv, int argc) { + int rret, wret; + char *old; + size_t sz = sizeof(old); + /* for strings, it seems we need to first get the old value, before overriding it. */ + if ((rret=je_mallctl(argv[0]->ptr, &old, &sz, NULL, 0))) { + /* return error unless this option is write only. */ + if (!(rret == EPERM && argc > 1)) { + addReplyErrorFormat(c,"%s", strerror(rret)); + return; + } + } + if(argc > 1) { + char *val = argv[1]->ptr; + char **valref = &val; + if ((!strcmp(val,"VOID"))) + valref = NULL, sz = 0; + wret = je_mallctl(argv[0]->ptr, NULL, 0, valref, sz); + } + if (!rret) + addReplyBulkCString(c, old); + else if (wret) + addReplyErrorFormat(c,"%s", strerror(wret)); + else + addReply(c, shared.ok); +} +#endif + void debugCommand(client *c) { if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { const char *help[] = { @@ -305,6 +375,7 @@ void debugCommand(client *c) { "CRASH-AND-RECOVER -- Hard crash and restart after delay.", "DIGEST -- Output a hex signature representing the current DB content.", "DIGEST-VALUE ... -- Output a hex signature of the values of all the specified keys.", +"DEBUG PROTOCOL [string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false]", "ERROR -- Return a Redis protocol error with as message. Useful for clients unit tests to simulate Redis errors.", "LOG -- write message to the server log.", "HTSTATS -- Return hash table statistics of the specified Redis database.", @@ -312,17 +383,23 @@ void debugCommand(client *c) { "LOADAOF -- Flush the AOF buffers on disk and reload the AOF in memory.", "LUA-ALWAYS-REPLICATE-COMMANDS <0|1> -- Setting it to 1 makes Lua replication defaulting to replicating single commands, without the script having to enable effects replication.", "OBJECT -- Show low level info about key and associated value.", +"OOM -- Crash the server simulating an out-of-memory error.", "PANIC -- Crash the server simulating a panic.", "POPULATE [prefix] [size] -- Create string keys named key:. If a prefix is specified is used instead of the 'key' prefix.", -"RELOAD -- Save the RDB on disk and reload it back in memory.", +"RELOAD [MERGE] [NOFLUSH] [NOSAVE] -- Save the RDB on disk and reload it back in memory. By default it will save the RDB file and load it back. With the NOFLUSH option the current database is not removed before loading the new one, but conficts in keys will kill the server with an exception. When MERGE is used, conflicting keys will be loaded (the key in the loaded RDB file will win). When NOSAVE is used, the server will not save the current dataset in the RDB file before loading. Use DEBUG RELOAD NOSAVE when you want just to load the RDB file you placed in the Redis working directory in order to replace the current dataset in memory. Use DEBUG RELOAD NOSAVE NOFLUSH MERGE when you want to add what is in the current RDB file placed in the Redis current directory, with the current memory content. Use DEBUG RELOAD when you want to verify Redis is able to persist the current dataset in the RDB file, flush the memory content, and load it back.", "RESTART -- Graceful restart: save config, db, restart.", "SDSLEN -- Show low level SDS string info representing key and value.", "SEGFAULT -- Crash the server with sigsegv.", "SET-ACTIVE-EXPIRE <0|1> -- Setting it to 0 disables expiring keys in background when they are not accessed (otherwise the Redis behavior). Setting it to 1 reenables back the default.", +"AOF-FLUSH-SLEEP -- Server will sleep before flushing the AOF, this is used for testing", "SLEEP -- Stop the server for . Decimals allowed.", "STRUCTSIZE -- Return the size of different Redis core C structures.", "ZIPLIST -- Show low level info about the ziplist encoding.", "STRINGMATCH-TEST -- Run a fuzz tester against the stringmatchlen() function.", +#ifdef USE_JEMALLOC +"MALLCTL [] -- Get or set a malloc tunning integer.", +"MALLCTL-STR [] -- Get or set a malloc tunning string.", +#endif NULL }; addReplyHelp(c, help); @@ -354,15 +431,44 @@ NULL serverLog(LL_WARNING, "DEBUG LOG: %s", (char*)c->argv[2]->ptr); addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"reload")) { - rdbSaveInfo rsi, *rsiptr; - rsiptr = rdbPopulateSaveInfo(&rsi); - if (rdbSave(server.rdb_filename,rsiptr) != C_OK) { - addReply(c,shared.err); - return; + int flush = 1, save = 1; + int flags = RDBFLAGS_NONE; + + /* Parse the additional options that modify the RELOAD + * behavior. */ + for (int j = 2; j < c->argc; j++) { + char *opt = c->argv[j]->ptr; + if (!strcasecmp(opt,"MERGE")) { + flags |= RDBFLAGS_ALLOW_DUP; + } else if (!strcasecmp(opt,"NOFLUSH")) { + flush = 0; + } else if (!strcasecmp(opt,"NOSAVE")) { + save = 0; + } else { + addReplyError(c,"DEBUG RELOAD only supports the " + "MERGE, NOFLUSH and NOSAVE options."); + return; + } } - emptyDb(-1,EMPTYDB_NO_FLAGS,NULL); + + /* The default beahvior is to save the RDB file before loading + * it back. */ + if (save) { + rdbSaveInfo rsi, *rsiptr; + rsiptr = rdbPopulateSaveInfo(&rsi); + if (rdbSave(server.rdb_filename,rsiptr) != C_OK) { + addReply(c,shared.err); + return; + } + } + + /* The default behavior is to remove the current dataset from + * memory before loading the RDB file, however when MERGE is + * used together with NOFLUSH, we are able to merge two datasets. */ + if (flush) emptyDb(-1,EMPTYDB_NO_FLAGS,NULL); + protectClient(c); - int ret = rdbLoad(server.rdb_filename,NULL); + int ret = rdbLoad(server.rdb_filename,NULL,flags); unprotectClient(c); if (ret != C_OK) { addReplyError(c,"Error trying to load the RDB dump"); @@ -433,7 +539,7 @@ NULL "encoding:%s serializedlength:%zu " "lru:%d lru_seconds_idle:%llu%s", (void*)val, val->refcount, - strenc, rdbSavedObjectLen(val), + strenc, rdbSavedObjectLen(val, c->argv[2]), val->lru, estimateObjectIdleTime(val)/1000, extra); } else if (!strcasecmp(c->argv[1]->ptr,"sdslen") && c->argc == 3) { dictEntry *de; @@ -502,7 +608,7 @@ NULL memcpy(val->ptr, buf, valsize<=buflen? valsize: buflen); } dbAdd(c->db,key,val); - signalModifiedKey(c->db,key); + signalModifiedKey(c,c->db,key); decrRefCount(key); } addReply(c,shared.ok); @@ -517,7 +623,7 @@ NULL sdsfree(d); } else if (!strcasecmp(c->argv[1]->ptr,"digest-value") && c->argc >= 2) { /* DEBUG DIGEST-VALUE key key key ... key. */ - addReplyMultiBulkLen(c,c->argc-2); + addReplyArrayLen(c,c->argc-2); for (int j = 2; j < c->argc; j++) { unsigned char digest[20]; memset(digest,0,20); /* Start with a clean result */ @@ -529,6 +635,58 @@ NULL addReplyStatus(c,d); sdsfree(d); } + } else if (!strcasecmp(c->argv[1]->ptr,"protocol") && c->argc == 3) { + /* DEBUG PROTOCOL [string|integer|double|bignum|null|array|set|map| + * attrib|push|verbatim|true|false] */ + char *name = c->argv[2]->ptr; + if (!strcasecmp(name,"string")) { + addReplyBulkCString(c,"Hello World"); + } else if (!strcasecmp(name,"integer")) { + addReplyLongLong(c,12345); + } else if (!strcasecmp(name,"double")) { + addReplyDouble(c,3.14159265359); + } else if (!strcasecmp(name,"bignum")) { + addReplyProto(c,"(1234567999999999999999999999999999999\r\n",40); + } else if (!strcasecmp(name,"null")) { + addReplyNull(c); + } else if (!strcasecmp(name,"array")) { + addReplyArrayLen(c,3); + for (int j = 0; j < 3; j++) addReplyLongLong(c,j); + } else if (!strcasecmp(name,"set")) { + addReplySetLen(c,3); + for (int j = 0; j < 3; j++) addReplyLongLong(c,j); + } else if (!strcasecmp(name,"map")) { + addReplyMapLen(c,3); + for (int j = 0; j < 3; j++) { + addReplyLongLong(c,j); + addReplyBool(c, j == 1); + } + } else if (!strcasecmp(name,"attrib")) { + addReplyAttributeLen(c,1); + addReplyBulkCString(c,"key-popularity"); + addReplyArrayLen(c,2); + addReplyBulkCString(c,"key:123"); + addReplyLongLong(c,90); + /* Attributes are not real replies, so a well formed reply should + * also have a normal reply type after the attribute. */ + addReplyBulkCString(c,"Some real reply following the attribute"); + } else if (!strcasecmp(name,"push")) { + addReplyPushLen(c,2); + addReplyBulkCString(c,"server-cpu-usage"); + addReplyLongLong(c,42); + /* Push replies are not synchronous replies, so we emit also a + * normal reply in order for blocking clients just discarding the + * push reply, to actually consume the reply and continue. */ + addReplyBulkCString(c,"Some real reply following the push reply"); + } else if (!strcasecmp(name,"true")) { + addReplyBool(c,1); + } else if (!strcasecmp(name,"false")) { + addReplyBool(c,0); + } else if (!strcasecmp(name,"verbatim")) { + addReplyVerbatim(c,"This is a verbatim\nstring",25,"txt"); + } else { + addReplyError(c,"Wrong protocol type name. Please use one of the following: string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false"); + } } else if (!strcasecmp(c->argv[1]->ptr,"sleep") && c->argc == 3) { double dtime = strtod(c->argv[2]->ptr,NULL); long long utime = dtime*1000000; @@ -543,6 +701,11 @@ NULL { server.active_expire_enabled = atoi(c->argv[2]->ptr); addReply(c,shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr,"aof-flush-sleep") && + c->argc == 3) + { + server.aof_flush_sleep = atoi(c->argv[2]->ptr); + addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"lua-always-replicate-commands") && c->argc == 3) { @@ -571,9 +734,12 @@ NULL sds stats = sdsempty(); char buf[4096]; - if (getLongFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK) + if (getLongFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK) { + sdsfree(stats); return; + } if (dbid < 0 || dbid >= server.dbnum) { + sdsfree(stats); addReplyError(c,"Out of range database"); return; } @@ -586,7 +752,8 @@ NULL dictGetStats(buf,sizeof(buf),server.db[dbid].expires); stats = sdscat(stats,buf); - addReplyBulkSds(c,stats); + addReplyVerbatim(c,stats,sdslen(stats),"txt"); + sdsfree(stats); } else if (!strcasecmp(c->argv[1]->ptr,"htstats-key") && c->argc == 3) { robj *o; dict *ht = NULL; @@ -613,7 +780,7 @@ NULL } else { char buf[4096]; dictGetStats(buf,sizeof(buf),ht); - addReplyBulkCString(c,buf); + addReplyVerbatim(c,buf,strlen(buf),"txt"); } } else if (!strcasecmp(c->argv[1]->ptr,"change-repl-id") && c->argc == 2) { serverLog(LL_WARNING,"Changing replication IDs after receiving DEBUG change-repl-id"); @@ -624,6 +791,14 @@ NULL { stringmatchlen_fuzz_test(); addReplyStatus(c,"Apparently Redis did not crash: test passed"); +#ifdef USE_JEMALLOC + } else if(!strcasecmp(c->argv[1]->ptr,"mallctl") && c->argc >= 3) { + mallctl_int(c, c->argv+2, c->argc-2); + return; + } else if(!strcasecmp(c->argv[1]->ptr,"mallctl-str") && c->argc >= 3) { + mallctl_string(c, c->argv+2, c->argc-2); + return; +#endif } else { addReplySubcommandSyntaxError(c); return; @@ -647,11 +822,12 @@ void _serverAssert(const char *estr, const char *file, int line) { void _serverAssertPrintClientInfo(const client *c) { int j; + char conninfo[CONN_INFO_LEN]; bugReportStart(); serverLog(LL_WARNING,"=== ASSERTION FAILED CLIENT CONTEXT ==="); - serverLog(LL_WARNING,"client->flags = %d", c->flags); - serverLog(LL_WARNING,"client->fd = %d", c->fd); + serverLog(LL_WARNING,"client->flags = %llu", (unsigned long long) c->flags); + serverLog(LL_WARNING,"client->conn = %s", connGetInfo(c->conn, conninfo, sizeof(conninfo))); serverLog(LL_WARNING,"client->argc = %d", c->argc); for (j=0; j < c->argc; j++) { char buf[128]; @@ -690,6 +866,8 @@ void serverLogObjectDebugInfo(const robj *o) { serverLog(LL_WARNING,"Sorted set size: %d", (int) zsetLength(o)); if (o->encoding == OBJ_ENCODING_SKIPLIST) serverLog(LL_WARNING,"Skiplist level: %d", (int) ((const zset*)o->ptr)->zsl->level); + } else if (o->type == OBJ_STREAM) { + serverLog(LL_WARNING,"Stream size: %d", (int) streamLength(o)); } } @@ -751,7 +929,7 @@ static void *getMcontextEip(ucontext_t *uc) { #endif #elif defined(__linux__) /* Linux */ - #if defined(__i386__) + #if defined(__i386__) || defined(__ILP32__) return (void*) uc->uc_mcontext.gregs[14]; /* Linux 32 */ #elif defined(__X86_64__) || defined(__x86_64__) return (void*) uc->uc_mcontext.gregs[16]; /* Linux 64 */ @@ -863,7 +1041,7 @@ void logRegisters(ucontext_t *uc) { /* Linux */ #elif defined(__linux__) /* Linux x86 */ - #if defined(__i386__) + #if defined(__i386__) || defined(__ILP32__) serverLog(LL_WARNING, "\n" "EAX:%08lx EBX:%08lx ECX:%08lx EDX:%08lx\n" @@ -918,6 +1096,61 @@ void logRegisters(ucontext_t *uc) { (unsigned long) uc->uc_mcontext.gregs[18] ); logStackContent((void**)uc->uc_mcontext.gregs[15]); + #elif defined(__aarch64__) /* Linux AArch64 */ + serverLog(LL_WARNING, + "\n" + "X18:%016lx X19:%016lx\nX20:%016lx X21:%016lx\n" + "X22:%016lx X23:%016lx\nX24:%016lx X25:%016lx\n" + "X26:%016lx X27:%016lx\nX28:%016lx X29:%016lx\n" + "X30:%016lx\n" + "pc:%016lx sp:%016lx\npstate:%016lx fault_address:%016lx\n", + (unsigned long) uc->uc_mcontext.regs[18], + (unsigned long) uc->uc_mcontext.regs[19], + (unsigned long) uc->uc_mcontext.regs[20], + (unsigned long) uc->uc_mcontext.regs[21], + (unsigned long) uc->uc_mcontext.regs[22], + (unsigned long) uc->uc_mcontext.regs[23], + (unsigned long) uc->uc_mcontext.regs[24], + (unsigned long) uc->uc_mcontext.regs[25], + (unsigned long) uc->uc_mcontext.regs[26], + (unsigned long) uc->uc_mcontext.regs[27], + (unsigned long) uc->uc_mcontext.regs[28], + (unsigned long) uc->uc_mcontext.regs[29], + (unsigned long) uc->uc_mcontext.regs[30], + (unsigned long) uc->uc_mcontext.pc, + (unsigned long) uc->uc_mcontext.sp, + (unsigned long) uc->uc_mcontext.pstate, + (unsigned long) uc->uc_mcontext.fault_address + ); + logStackContent((void**)uc->uc_mcontext.sp); + #elif defined(__arm__) /* Linux ARM */ + serverLog(LL_WARNING, + "\n" + "R10:%016lx R9 :%016lx\nR8 :%016lx R7 :%016lx\n" + "R6 :%016lx R5 :%016lx\nR4 :%016lx R3 :%016lx\n" + "R2 :%016lx R1 :%016lx\nR0 :%016lx EC :%016lx\n" + "fp: %016lx ip:%016lx\n", + "pc:%016lx sp:%016lx\ncpsr:%016lx fault_address:%016lx\n", + (unsigned long) uc->uc_mcontext.arm_r10, + (unsigned long) uc->uc_mcontext.arm_r9, + (unsigned long) uc->uc_mcontext.arm_r8, + (unsigned long) uc->uc_mcontext.arm_r7, + (unsigned long) uc->uc_mcontext.arm_r6, + (unsigned long) uc->uc_mcontext.arm_r5, + (unsigned long) uc->uc_mcontext.arm_r4, + (unsigned long) uc->uc_mcontext.arm_r3, + (unsigned long) uc->uc_mcontext.arm_r2, + (unsigned long) uc->uc_mcontext.arm_r1, + (unsigned long) uc->uc_mcontext.arm_r0, + (unsigned long) uc->uc_mcontext.error_code, + (unsigned long) uc->uc_mcontext.arm_fp, + (unsigned long) uc->uc_mcontext.arm_ip, + (unsigned long) uc->uc_mcontext.arm_pc, + (unsigned long) uc->uc_mcontext.arm_sp, + (unsigned long) uc->uc_mcontext.arm_cpsr, + (unsigned long) uc->uc_mcontext.fault_address + ); + logStackContent((void**)uc->uc_mcontext.arm_sp); #endif #elif defined(__FreeBSD__) #if defined(__x86_64__) @@ -1285,6 +1518,12 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { /* Log dump of processor registers */ logRegisters(uc); + /* Log Modules INFO */ + serverLogRaw(LL_WARNING|LL_RAW, "\n------ MODULES INFO OUTPUT ------\n"); + infostring = modulesCollectInfo(sdsempty(), NULL, 1, 0); + serverLogRaw(LL_WARNING|LL_RAW, infostring); + sdsfree(infostring); + #if defined(HAVE_PROC_MAPS) /* Test memory */ serverLogRaw(LL_WARNING|LL_RAW, "\n------ FAST MEMORY TEST ------\n"); @@ -1417,7 +1656,7 @@ void enableWatchdog(int period) { /* Watchdog was actually disabled, so we have to setup the signal * handler. */ sigemptyset(&act.sa_mask); - act.sa_flags = SA_ONSTACK | SA_SIGINFO; + act.sa_flags = SA_SIGINFO; act.sa_sigaction = watchdogSignalHandler; sigaction(SIGALRM, &act, NULL); } diff --git a/redis.submodule/src/defrag.c b/redis.submodule/src/defrag.c index ecf0255..6e52966 100644 --- a/redis.submodule/src/defrag.c +++ b/redis.submodule/src/defrag.c @@ -5,8 +5,8 @@ * We do that by scanning the keyspace and for each pointer we have, we can try to * ask the allocator if moving it to a new address will help reduce fragmentation. * - * Copyright (c) 2017, Oran Agra - * Copyright (c) 2017, Redis Labs, Inc + * Copyright (c) 2020, Oran Agra + * Copyright (c) 2020, Redis Labs, Inc * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -43,7 +43,7 @@ /* this method was added to jemalloc in order to help us understand which * pointers are worthwhile moving and which aren't */ -int je_get_defrag_hint(void* ptr, int *bin_util, int *run_util); +int je_get_defrag_hint(void* ptr); /* forward declarations*/ void defragDictBucketCallback(void *privdata, dictEntry **bucketref); @@ -55,18 +55,11 @@ dictEntry* replaceSateliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sd * when it returns a non-null value, the old pointer was already released * and should NOT be accessed. */ void* activeDefragAlloc(void *ptr) { - int bin_util, run_util; size_t size; void *newptr; - if(!je_get_defrag_hint(ptr, &bin_util, &run_util)) { - server.stat_active_defrag_misses++; - return NULL; - } - /* if this run is more utilized than the average utilization in this bin - * (or it is full), skip it. This will eventually move all the allocations - * from relatively empty runs into relatively full runs. */ - if (run_util > bin_util || run_util == 1<<16) { + if(!je_get_defrag_hint(ptr)) { server.stat_active_defrag_misses++; + size = zmalloc_size(ptr); return NULL; } /* move this allocation to a new allocation. @@ -374,7 +367,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) { if ((newele = activeDefragStringOb(ele, &defragged))) de->v.val = newele, defragged++; } else if (dict_val_type == DEFRAG_SDS_DICT_VAL_VOID_PTR) { - void *newptr, *ptr = ln->value; + void *newptr, *ptr = dictGetVal(de); if ((newptr = activeDefragAlloc(ptr))) ln->value = newptr, defragged++; } @@ -408,25 +401,32 @@ dictEntry* replaceSateliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sd return NULL; } -long activeDefragQuickListNodes(quicklist *ql) { - quicklistNode *node = ql->head, *newnode; +long activeDefragQuickListNode(quicklist *ql, quicklistNode **node_ref) { + quicklistNode *newnode, *node = *node_ref; long defragged = 0; unsigned char *newzl; + if ((newnode = activeDefragAlloc(node))) { + if (newnode->prev) + newnode->prev->next = newnode; + else + ql->head = newnode; + if (newnode->next) + newnode->next->prev = newnode; + else + ql->tail = newnode; + *node_ref = node = newnode; + defragged++; + } + if ((newzl = activeDefragAlloc(node->zl))) + defragged++, node->zl = newzl; + return defragged; +} + +long activeDefragQuickListNodes(quicklist *ql) { + quicklistNode *node = ql->head; + long defragged = 0; while (node) { - if ((newnode = activeDefragAlloc(node))) { - if (newnode->prev) - newnode->prev->next = newnode; - else - ql->head = newnode; - if (newnode->next) - newnode->next->prev = newnode; - else - ql->tail = newnode; - node = newnode; - defragged++; - } - if ((newzl = activeDefragAlloc(node->zl))) - defragged++, node->zl = newzl; + defragged += activeDefragQuickListNode(ql, &node); node = node->next; } return defragged; @@ -440,12 +440,48 @@ void defragLater(redisDb *db, dictEntry *kde) { listAddNodeTail(db->defrag_later, key); } -long scanLaterList(robj *ob) { +/* returns 0 if no more work needs to be been done, and 1 if time is up and more work is needed. */ +long scanLaterList(robj *ob, unsigned long *cursor, long long endtime, long long *defragged) { quicklist *ql = ob->ptr; + quicklistNode *node; + long iterations = 0; + int bookmark_failed = 0; if (ob->type != OBJ_LIST || ob->encoding != OBJ_ENCODING_QUICKLIST) return 0; - server.stat_active_defrag_scanned+=ql->len; - return activeDefragQuickListNodes(ql); + + if (*cursor == 0) { + /* if cursor is 0, we start new iteration */ + node = ql->head; + } else { + node = quicklistBookmarkFind(ql, "_AD"); + if (!node) { + /* if the bookmark was deleted, it means we reached the end. */ + *cursor = 0; + return 0; + } + node = node->next; + } + + (*cursor)++; + while (node) { + (*defragged) += activeDefragQuickListNode(ql, &node); + server.stat_active_defrag_scanned++; + if (++iterations > 128 && !bookmark_failed) { + if (ustime() > endtime) { + if (!quicklistBookmarkCreate(&ql, "_AD", node)) { + bookmark_failed = 1; + } else { + ob->ptr = ql; /* bookmark creation may have re-allocated the quicklist */ + return 1; + } + } + iterations = 0; + } + node = node->next; + } + quicklistBookmarkDelete(ql, "_AD"); + *cursor = 0; + return bookmark_failed? 1: 0; } typedef struct { @@ -638,7 +674,8 @@ int scanLaterStraemListpacks(robj *ob, unsigned long *cursor, long long endtime, void *newdata = activeDefragAlloc(ri.data); if (newdata) raxSetData(ri.node, ri.data=newdata), (*defragged)++; - if (++iterations > 16) { + server.stat_active_defrag_scanned++; + if (++iterations > 128) { if (ustime() > endtime) { serverAssert(ri.key_len==sizeof(last)); memcpy(last,ri.key,ri.key_len); @@ -900,8 +937,7 @@ int defragLaterItem(dictEntry *de, unsigned long *cursor, long long endtime) { if (de) { robj *ob = dictGetVal(de); if (ob->type == OBJ_LIST) { - server.stat_active_defrag_hits += scanLaterList(ob); - *cursor = 0; /* list has no scan, we must finish it in one go */ + return scanLaterList(ob, cursor, endtime, &server.stat_active_defrag_hits); } else if (ob->type == OBJ_SET) { server.stat_active_defrag_hits += scanLaterSet(ob, cursor); } else if (ob->type == OBJ_ZSET) { @@ -919,10 +955,12 @@ int defragLaterItem(dictEntry *de, unsigned long *cursor, long long endtime) { return 0; } +/* static variables serving defragLaterStep to continue scanning a key from were we stopped last time. */ +static sds defrag_later_current_key = NULL; +static unsigned long defrag_later_cursor = 0; + /* returns 0 if no more work needs to be been done, and 1 if time is up and more work is needed. */ int defragLaterStep(redisDb *db, long long endtime) { - static sds current_key = NULL; - static unsigned long cursor = 0; unsigned int iterations = 0; unsigned long long prev_defragged = server.stat_active_defrag_hits; unsigned long long prev_scanned = server.stat_active_defrag_scanned; @@ -930,16 +968,15 @@ int defragLaterStep(redisDb *db, long long endtime) { do { /* if we're not continuing a scan from the last call or loop, start a new one */ - if (!cursor) { + if (!defrag_later_cursor) { listNode *head = listFirst(db->defrag_later); /* Move on to next key */ - if (current_key) { - serverAssert(current_key == head->value); - sdsfree(head->value); + if (defrag_later_current_key) { + serverAssert(defrag_later_current_key == head->value); listDelNode(db->defrag_later, head); - cursor = 0; - current_key = NULL; + defrag_later_cursor = 0; + defrag_later_current_key = NULL; } /* stop if we reached the last one. */ @@ -948,23 +985,18 @@ int defragLaterStep(redisDb *db, long long endtime) { return 0; /* start a new key */ - current_key = head->value; - cursor = 0; + defrag_later_current_key = head->value; + defrag_later_cursor = 0; } /* each time we enter this function we need to fetch the key from the dict again (if it still exists) */ - dictEntry *de = dictFind(db->dict, current_key); + dictEntry *de = dictFind(db->dict, defrag_later_current_key); key_defragged = server.stat_active_defrag_hits; do { int quit = 0; - if (defragLaterItem(de, &cursor, endtime)) + if (defragLaterItem(de, &defrag_later_cursor, endtime)) quit = 1; /* time is up, we didn't finish all the work */ - /* Don't start a new BIG key in this loop, this is because the - * next key can be a list, and scanLaterList must be done in once cycle */ - if (!cursor) - quit = 1; - /* Once in 16 scan iterations, 512 pointer reallocations, or 64 fields * (if we have a lot of pointers in one hash bucket, or rehashing), * check if we reached the time limit. */ @@ -982,7 +1014,7 @@ int defragLaterStep(redisDb *db, long long endtime) { prev_defragged = server.stat_active_defrag_hits; prev_scanned = server.stat_active_defrag_scanned; } - } while(cursor); + } while(defrag_later_cursor); if(key_defragged != server.stat_active_defrag_hits) server.stat_active_defrag_key_hits++; else @@ -1039,7 +1071,22 @@ void activeDefragCycle(void) { mstime_t latency; int quit = 0; - if (server.aof_child_pid!=-1 || server.rdb_child_pid!=-1) + if (!server.active_defrag_enabled) { + if (server.active_defrag_running) { + /* if active defrag was disabled mid-run, start from fresh next time. */ + server.active_defrag_running = 0; + if (db) + listEmpty(db->defrag_later); + defrag_later_current_key = NULL; + defrag_later_cursor = 0; + current_db = -1; + cursor = 0; + db = NULL; + } + return; + } + + if (hasActiveChildProcess()) return; /* Defragging memory while there's a fork will just do damage. */ /* Once a second, check if we the fragmentation justfies starting a scan diff --git a/redis.submodule/src/dict.c b/redis.submodule/src/dict.c index 2cf9d48..45aab66 100644 --- a/redis.submodule/src/dict.c +++ b/redis.submodule/src/dict.c @@ -134,7 +134,7 @@ int _dictInit(dict *d, dictType *type, * but with the invariant of a USED/BUCKETS ratio near to <= 1 */ int dictResize(dict *d) { - int minimal; + unsigned long minimal; if (!dict_can_resize || dictIsRehashing(d)) return DICT_ERR; minimal = d->ht[0].used; @@ -478,7 +478,7 @@ dictEntry *dictFind(dict *d, const void *key) dictEntry *he; uint64_t h, idx, table; - if (d->ht[0].used + d->ht[1].used == 0) return NULL; /* dict is empty */ + if (dictSize(d) == 0) return NULL; /* dict is empty */ if (dictIsRehashing(d)) _dictRehashStep(d); h = dictHashKey(d, key); for (table = 0; table <= 1; table++) { @@ -739,11 +739,35 @@ unsigned int dictGetSomeKeys(dict *d, dictEntry **des, unsigned int count) { return stored; } +/* This is like dictGetRandomKey() from the POV of the API, but will do more + * work to ensure a better distribution of the returned element. + * + * This function improves the distribution because the dictGetRandomKey() + * problem is that it selects a random bucket, then it selects a random + * element from the chain in the bucket. However elements being in different + * chain lengths will have different probabilities of being reported. With + * this function instead what we do is to consider a "linear" range of the table + * that may be constituted of N buckets with chains of different lengths + * appearing one after the other. Then we report a random element in the range. + * In this way we smooth away the problem of different chain lenghts. */ +#define GETFAIR_NUM_ENTRIES 15 +dictEntry *dictGetFairRandomKey(dict *d) { + dictEntry *entries[GETFAIR_NUM_ENTRIES]; + unsigned int count = dictGetSomeKeys(d,entries,GETFAIR_NUM_ENTRIES); + /* Note that dictGetSomeKeys() may return zero elements in an unlucky + * run() even if there are actually elements inside the hash table. So + * when we get zero, we call the true dictGetRandomKey() that will always + * yeld the element if the hash table has at least one. */ + if (count == 0) return dictGetRandomKey(d); + unsigned int idx = rand() % count; + return entries[idx]; +} + /* Function to reverse bits. Algorithm from: * http://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel */ static unsigned long rev(unsigned long v) { - unsigned long s = 8 * sizeof(v); // bit size; must be power of 2 - unsigned long mask = ~0; + unsigned long s = CHAR_BIT * sizeof(v); // bit size; must be power of 2 + unsigned long mask = ~0UL; while ((s >>= 1) > 0) { mask ^= (mask << s); v = ((v >> s) & mask) | ((v << s) & ~mask); @@ -847,6 +871,10 @@ unsigned long dictScan(dict *d, if (dictSize(d) == 0) return 0; + /* Having a safe iterator means no rehashing can happen, see _dictRehashStep. + * This is needed in case the scan callback tries to do dictFind or alike. */ + d->iterators++; + if (!dictIsRehashing(d)) { t0 = &(d->ht[0]); m0 = t0->sizemask; @@ -913,6 +941,9 @@ unsigned long dictScan(dict *d, } while (v & (m0 ^ m1)); } + /* undo the ++ at the top */ + d->iterators--; + return v; } @@ -1013,7 +1044,7 @@ dictEntry **dictFindEntryRefByPtrAndHash(dict *d, const void *oldptr, uint64_t h dictEntry *he, **heref; unsigned long idx, table; - if (d->ht[0].used + d->ht[1].used == 0) return NULL; /* dict is empty */ + if (dictSize(d) == 0) return NULL; /* dict is empty */ for (table = 0; table <= 1; table++) { idx = hash & d->ht[table].sizemask; heref = &d->ht[table].table[idx]; diff --git a/redis.submodule/src/dict.h b/redis.submodule/src/dict.h index 62018cc..dec60f6 100644 --- a/redis.submodule/src/dict.h +++ b/redis.submodule/src/dict.h @@ -166,6 +166,7 @@ dictIterator *dictGetSafeIterator(dict *d); dictEntry *dictNext(dictIterator *iter); void dictReleaseIterator(dictIterator *iter); dictEntry *dictGetRandomKey(dict *d); +dictEntry *dictGetFairRandomKey(dict *d); unsigned int dictGetSomeKeys(dict *d, dictEntry **des, unsigned int count); void dictGetStats(char *buf, size_t bufsize, dict *d); uint64_t dictGenHashFunction(const void *key, int len); diff --git a/redis.submodule/src/evict.c b/redis.submodule/src/evict.c index 773916c..0755acc 100644 --- a/redis.submodule/src/evict.c +++ b/redis.submodule/src/evict.c @@ -78,7 +78,7 @@ unsigned int getLRUClock(void) { unsigned int LRU_CLOCK(void) { unsigned int lruclock; if (1000/server.hz <= LRU_CLOCK_RESOLUTION) { - atomicGet(server.lruclock,lruclock); + lruclock = server.lruclock; } else { lruclock = getLRUClock(); } @@ -444,14 +444,16 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev * Otehrwise if we are over the memory limit, but not enough memory * was freed to return back under the limit, the function returns C_ERR. */ int freeMemoryIfNeeded(void) { + int keys_freed = 0; /* By default replicas should ignore maxmemory * and just be masters exact copies. */ if (server.masterhost && server.repl_slave_ignore_maxmemory) return C_OK; size_t mem_reported, mem_tofree, mem_freed; - mstime_t latency, eviction_latency; + mstime_t latency, eviction_latency, lazyfree_latency; long long delta; int slaves = listLength(server.slaves); + int result = C_ERR; /* When clients are paused the dataset should be static not just from the * POV of clients not being able to write, but also from the POV of @@ -462,12 +464,12 @@ int freeMemoryIfNeeded(void) { mem_freed = 0; + latencyStartMonitor(latency); if (server.maxmemory_policy == MAXMEMORY_NO_EVICTION) goto cant_free; /* We need to free memory, but policy forbids. */ - latencyStartMonitor(latency); while (mem_freed < mem_tofree) { - int j, k, i, keys_freed = 0; + int j, k, i; static unsigned int next_db = 0; sds bestkey = NULL; int bestdbid; @@ -568,9 +570,9 @@ int freeMemoryIfNeeded(void) { dbAsyncDelete(db,keyobj); else dbSyncDelete(db,keyobj); + signalModifiedKey(NULL,db,keyobj); latencyEndMonitor(eviction_latency); latencyAddSampleIfNeeded("eviction-del",eviction_latency); - latencyRemoveNestedEvent(latency,eviction_latency); delta -= (long long) zmalloc_used_memory(); mem_freed += delta; server.stat_evictedkeys++; @@ -598,28 +600,31 @@ int freeMemoryIfNeeded(void) { mem_freed = mem_tofree; } } - } - - if (!keys_freed) { - latencyEndMonitor(latency); - latencyAddSampleIfNeeded("eviction-cycle",latency); + } else { goto cant_free; /* nothing to free... */ } } - latencyEndMonitor(latency); - latencyAddSampleIfNeeded("eviction-cycle",latency); - return C_OK; + result = C_OK; cant_free: /* We are here if we are not able to reclaim memory. There is only one * last thing we can try: check if the lazyfree thread has jobs in queue * and wait... */ - while(bioPendingJobsOfType(BIO_LAZY_FREE)) { - if (((mem_reported - zmalloc_used_memory()) + mem_freed) >= mem_tofree) - break; - usleep(1000); + if (result != C_OK) { + latencyStartMonitor(lazyfree_latency); + while(bioPendingJobsOfType(BIO_LAZY_FREE)) { + if (getMaxmemoryState(NULL,NULL,NULL,NULL) == C_OK) { + result = C_OK; + break; + } + usleep(1000); + } + latencyEndMonitor(lazyfree_latency); + latencyAddSampleIfNeeded("eviction-lazyfree",lazyfree_latency); } - return C_ERR; + latencyEndMonitor(latency); + latencyAddSampleIfNeeded("eviction-cycle",latency); + return result; } /* This is a wrapper for freeMemoryIfNeeded() that only really calls the diff --git a/redis.submodule/src/expire.c b/redis.submodule/src/expire.c index 0b92ee3..30a2719 100644 --- a/redis.submodule/src/expire.c +++ b/redis.submodule/src/expire.c @@ -64,6 +64,7 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) { dbSyncDelete(db,keyobj); notifyKeyspaceEvent(NOTIFY_EXPIRED, "expired",keyobj,db->id); + trackingInvalidateKey(NULL,keyobj); decrRefCount(keyobj); server.stat_expiredkeys++; return 1; @@ -77,24 +78,63 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) { * it will get more aggressive to avoid that too much memory is used by * keys that can be removed from the keyspace. * - * No more than CRON_DBS_PER_CALL databases are tested at every - * iteration. + * Every expire cycle tests multiple databases: the next call will start + * again from the next db, with the exception of exists for time limit: in that + * case we restart again from the last database we were processing. Anyway + * no more than CRON_DBS_PER_CALL databases are tested at every iteration. * - * This kind of call is used when Redis detects that timelimit_exit is - * true, so there is more work to do, and we do it more incrementally from - * the beforeSleep() function of the event loop. + * The function can perform more or less work, depending on the "type" + * argument. It can execute a "fast cycle" or a "slow cycle". The slow + * cycle is the main way we collect expired cycles: this happens with + * the "server.hz" frequency (usually 10 hertz). * - * Expire cycle type: + * However the slow cycle can exit for timeout, since it used too much time. + * For this reason the function is also invoked to perform a fast cycle + * at every event loop cycle, in the beforeSleep() function. The fast cycle + * will try to perform less work, but will do it much more often. + * + * The following are the details of the two expire cycles and their stop + * conditions: * * If type is ACTIVE_EXPIRE_CYCLE_FAST the function will try to run a * "fast" expire cycle that takes no longer than EXPIRE_FAST_CYCLE_DURATION * microseconds, and is not repeated again before the same amount of time. + * The cycle will also refuse to run at all if the latest slow cycle did not + * terminate because of a time limit condition. * * If type is ACTIVE_EXPIRE_CYCLE_SLOW, that normal expire cycle is * executed, where the time limit is a percentage of the REDIS_HZ period - * as specified by the ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC define. */ + * as specified by the ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC define. In the + * fast cycle, the check of every database is interrupted once the number + * of already expired keys in the database is estimated to be lower than + * a given percentage, in order to avoid doing too much work to gain too + * little memory. + * + * The configured expire "effort" will modify the baseline parameters in + * order to do more work in both the fast and slow expire cycles. + */ + +#define ACTIVE_EXPIRE_CYCLE_KEYS_PER_LOOP 20 /* Keys for each DB loop. */ +#define ACTIVE_EXPIRE_CYCLE_FAST_DURATION 1000 /* Microseconds. */ +#define ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC 25 /* Max % of CPU to use. */ +#define ACTIVE_EXPIRE_CYCLE_ACCEPTABLE_STALE 10 /* % of stale keys after which + we do extra efforts. */ void activeExpireCycle(int type) { + /* Adjust the running parameters according to the configured expire + * effort. The default effort is 1, and the maximum configurable effort + * is 10. */ + unsigned long + effort = server.active_expire_effort-1, /* Rescale from 0 to 9. */ + config_keys_per_loop = ACTIVE_EXPIRE_CYCLE_KEYS_PER_LOOP + + ACTIVE_EXPIRE_CYCLE_KEYS_PER_LOOP/4*effort, + config_cycle_fast_duration = ACTIVE_EXPIRE_CYCLE_FAST_DURATION + + ACTIVE_EXPIRE_CYCLE_FAST_DURATION/4*effort, + config_cycle_slow_time_perc = ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC + + 2*effort, + config_cycle_acceptable_stale = ACTIVE_EXPIRE_CYCLE_ACCEPTABLE_STALE- + effort; + /* This function has some global state in order to continue the work * incrementally across calls. */ static unsigned int current_db = 0; /* Last DB tested. */ @@ -112,10 +152,16 @@ void activeExpireCycle(int type) { if (type == ACTIVE_EXPIRE_CYCLE_FAST) { /* Don't start a fast cycle if the previous cycle did not exit - * for time limit. Also don't repeat a fast cycle for the same period + * for time limit, unless the percentage of estimated stale keys is + * too high. Also never repeat a fast cycle for the same period * as the fast cycle total duration itself. */ - if (!timelimit_exit) return; - if (start < last_fast_cycle + ACTIVE_EXPIRE_CYCLE_FAST_DURATION*2) return; + if (!timelimit_exit && + server.stat_expired_stale_perc < config_cycle_acceptable_stale) + return; + + if (start < last_fast_cycle + (long long)config_cycle_fast_duration*2) + return; + last_fast_cycle = start; } @@ -129,16 +175,16 @@ void activeExpireCycle(int type) { if (dbs_per_call > server.dbnum || timelimit_exit) dbs_per_call = server.dbnum; - /* We can use at max ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC percentage of CPU time - * per iteration. Since this function gets called with a frequency of + /* We can use at max 'config_cycle_slow_time_perc' percentage of CPU + * time per iteration. Since this function gets called with a frequency of * server.hz times per second, the following is the max amount of * microseconds we can spend in this function. */ - timelimit = 1000000*ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC/server.hz/100; + timelimit = config_cycle_slow_time_perc*1000000/server.hz/100; timelimit_exit = 0; if (timelimit <= 0) timelimit = 1; if (type == ACTIVE_EXPIRE_CYCLE_FAST) - timelimit = ACTIVE_EXPIRE_CYCLE_FAST_DURATION; /* in microseconds. */ + timelimit = config_cycle_fast_duration; /* in microseconds. */ /* Accumulate some global stats as we expire keys, to have some idea * about the number of keys that are already logically expired, but still @@ -147,7 +193,9 @@ void activeExpireCycle(int type) { long total_expired = 0; for (j = 0; j < dbs_per_call && timelimit_exit == 0; j++) { - int expired; + /* Expired and checked in a single loop. */ + unsigned long expired, sampled; + redisDb *db = server.db+(current_db % server.dbnum); /* Increment the DB now so we are sure if we run out of time @@ -155,8 +203,10 @@ void activeExpireCycle(int type) { * distribute the time evenly across DBs. */ current_db++; - /* Continue to expire if at the end of the cycle more than 25% - * of the keys were expired. */ + /* Continue to expire if at the end of the cycle there are still + * a big percentage of keys to expire, compared to the number of keys + * we scanned. The percentage, stored in config_cycle_acceptable_stale + * is not fixed, but depends on the Redis configured "expire effort". */ do { unsigned long num, slots; long long now, ttl_sum; @@ -171,8 +221,8 @@ void activeExpireCycle(int type) { slots = dictSlots(db->expires); now = mstime(); - /* When there are less than 1% filled slots getting random - * keys is expensive, so stop here waiting for better times... + /* When there are less than 1% filled slots, sampling the key + * space is expensive, so stop here waiting for better times... * The dictionary will be resized asap. */ if (num && slots > DICT_HT_INITIAL_SIZE && (num*100/slots < 1)) break; @@ -180,27 +230,58 @@ void activeExpireCycle(int type) { /* The main collection cycle. Sample random keys among keys * with an expire set, checking for expired ones. */ expired = 0; + sampled = 0; ttl_sum = 0; ttl_samples = 0; - if (num > ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP) - num = ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP; - - while (num--) { - dictEntry *de; - long long ttl; - - if ((de = dictGetRandomKey(db->expires)) == NULL) break; - ttl = dictGetSignedIntegerVal(de)-now; - if (activeExpireCycleTryExpire(db,de,now)) expired++; - if (ttl > 0) { - /* We want the average TTL of keys yet not expired. */ - ttl_sum += ttl; - ttl_samples++; + if (num > config_keys_per_loop) + num = config_keys_per_loop; + + /* Here we access the low level representation of the hash table + * for speed concerns: this makes this code coupled with dict.c, + * but it hardly changed in ten years. + * + * Note that certain places of the hash table may be empty, + * so we want also a stop condition about the number of + * buckets that we scanned. However scanning for free buckets + * is very fast: we are in the cache line scanning a sequential + * array of NULL pointers, so we can scan a lot more buckets + * than keys in the same time. */ + long max_buckets = num*20; + long checked_buckets = 0; + + while (sampled < num && checked_buckets < max_buckets) { + for (int table = 0; table < 2; table++) { + if (table == 1 && !dictIsRehashing(db->expires)) break; + + unsigned long idx = db->expires_cursor; + idx &= db->expires->ht[table].sizemask; + dictEntry *de = db->expires->ht[table].table[idx]; + long long ttl; + + /* Scan the current bucket of the current table. */ + checked_buckets++; + while(de) { + /* Get the next entry now since this entry may get + * deleted. */ + dictEntry *e = de; + de = de->next; + + ttl = dictGetSignedIntegerVal(e)-now; + if (activeExpireCycleTryExpire(db,e,now)) expired++; + if (ttl > 0) { + /* We want the average TTL of keys yet + * not expired. */ + ttl_sum += ttl; + ttl_samples++; + } + sampled++; + } } - total_sampled++; + db->expires_cursor++; } total_expired += expired; + total_sampled += sampled; /* Update the average TTL stats for this database. */ if (ttl_samples) { @@ -224,12 +305,15 @@ void activeExpireCycle(int type) { break; } } - /* We don't repeat the cycle if there are less than 25% of keys - * found expired in the current DB. */ - } while (expired > ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP/4); + /* We don't repeat the cycle for the current database if there are + * an acceptable amount of stale keys (logically expired but yet + * not reclaimed). */ + } while (sampled == 0 || + (expired*100/sampled) > config_cycle_acceptable_stale); } elapsed = ustime()-start; + server.stat_expire_cycle_time_used += elapsed; latencyAddSampleIfNeeded("expire-cycle",elapsed/1000); /* Update our estimate of keys existing but yet to be expired. @@ -435,14 +519,14 @@ void expireGenericCommand(client *c, long long basetime, int unit) { /* Replicate/AOF this as an explicit DEL or UNLINK. */ aux = server.lazyfree_lazy_expire ? shared.unlink : shared.del; rewriteClientCommandVector(c,2,aux,key); - signalModifiedKey(c->db,key); + signalModifiedKey(c,c->db,key); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id); addReply(c, shared.cone); return; } else { setExpire(c,c->db,key,when); addReply(c,shared.cone); - signalModifiedKey(c->db,key); + signalModifiedKey(c,c->db,key); notifyKeyspaceEvent(NOTIFY_GENERIC,"expire",key,c->db->id); server.dirty++; return; @@ -506,6 +590,7 @@ void pttlCommand(client *c) { void persistCommand(client *c) { if (lookupKeyWrite(c->db,c->argv[1])) { if (removeExpire(c->db,c->argv[1])) { + notifyKeyspaceEvent(NOTIFY_GENERIC,"persist",c->argv[1],c->db->id); addReply(c,shared.cone); server.dirty++; } else { diff --git a/redis.submodule/src/geo.c b/redis.submodule/src/geo.c index f1d3f18..3e5d5f6 100644 --- a/redis.submodule/src/geo.c +++ b/redis.submodule/src/geo.c @@ -466,7 +466,7 @@ void georadiusGeneric(client *c, int flags) { /* Look up the requested zset */ robj *zobj = NULL; - if ((zobj = lookupKeyReadOrReply(c, key, shared.emptymultibulk)) == NULL || + if ((zobj = lookupKeyReadOrReply(c, key, shared.emptyarray)) == NULL || checkType(c, zobj, OBJ_ZSET)) { return; } @@ -566,7 +566,7 @@ void georadiusGeneric(client *c, int flags) { /* If no matching results, the user gets an empty reply. */ if (ga->used == 0 && storekey == NULL) { - addReply(c, shared.emptymultibulk); + addReply(c,shared.emptyarray); geoArrayFree(ga); return; } @@ -597,11 +597,11 @@ void georadiusGeneric(client *c, int flags) { if (withhash) option_length++; - /* The multibulk len we send is exactly result_length. The result is + /* The array len we send is exactly result_length. The result is * either all strings of just zset members *or* a nested multi-bulk * reply containing the zset member string _and_ all the additional * options the user enabled for this request. */ - addReplyMultiBulkLen(c, returned_items); + addReplyArrayLen(c, returned_items); /* Finally send results back to the caller */ int i; @@ -613,7 +613,7 @@ void georadiusGeneric(client *c, int flags) { * as a nested multi-bulk. Add 1 to account for result value * itself. */ if (option_length) - addReplyMultiBulkLen(c, option_length + 1); + addReplyArrayLen(c, option_length + 1); addReplyBulkSds(c,gp->member); gp->member = NULL; @@ -625,7 +625,7 @@ void georadiusGeneric(client *c, int flags) { addReplyLongLong(c, gp->score); if (withcoords) { - addReplyMultiBulkLen(c, 2); + addReplyArrayLen(c, 2); addReplyHumanLongDouble(c, gp->longitude); addReplyHumanLongDouble(c, gp->latitude); } @@ -657,13 +657,13 @@ void georadiusGeneric(client *c, int flags) { if (returned_items) { zsetConvertToZiplistIfNeeded(zobj,maxelelen); - setKey(c->db,storekey,zobj); + setKey(c,c->db,storekey,zobj); decrRefCount(zobj); notifyKeyspaceEvent(NOTIFY_ZSET,"georadiusstore",storekey, c->db->id); server.dirty += returned_items; } else if (dbDelete(c->db,storekey)) { - signalModifiedKey(c->db,storekey); + signalModifiedKey(c,c->db,storekey); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",storekey,c->db->id); server.dirty++; } @@ -706,11 +706,11 @@ void geohashCommand(client *c) { /* Geohash elements one after the other, using a null bulk reply for * missing elements. */ - addReplyMultiBulkLen(c,c->argc-2); + addReplyArrayLen(c,c->argc-2); for (j = 2; j < c->argc; j++) { double score; if (!zobj || zsetScore(zobj, c->argv[j]->ptr, &score) == C_ERR) { - addReply(c,shared.nullbulk); + addReplyNull(c); } else { /* The internal format we use for geocoding is a bit different * than the standard, since we use as initial latitude range @@ -721,7 +721,7 @@ void geohashCommand(client *c) { /* Decode... */ double xy[2]; if (!decodeGeohash(score,xy)) { - addReply(c,shared.nullbulk); + addReplyNull(c); continue; } @@ -737,7 +737,15 @@ void geohashCommand(client *c) { char buf[12]; int i; for (i = 0; i < 11; i++) { - int idx = (hash.bits >> (52-((i+1)*5))) & 0x1f; + int idx; + if (i == 10) { + /* We have just 52 bits, but the API used to output + * an 11 bytes geohash. For compatibility we assume + * zero. */ + idx = 0; + } else { + idx = (hash.bits >> (52-((i+1)*5))) & 0x1f; + } buf[i] = geoalphabet[idx]; } buf[11] = '\0'; @@ -759,19 +767,19 @@ void geoposCommand(client *c) { /* Report elements one after the other, using a null bulk reply for * missing elements. */ - addReplyMultiBulkLen(c,c->argc-2); + addReplyArrayLen(c,c->argc-2); for (j = 2; j < c->argc; j++) { double score; if (!zobj || zsetScore(zobj, c->argv[j]->ptr, &score) == C_ERR) { - addReply(c,shared.nullmultibulk); + addReplyNullArray(c); } else { /* Decode... */ double xy[2]; if (!decodeGeohash(score,xy)) { - addReply(c,shared.nullmultibulk); + addReplyNullArray(c); continue; } - addReplyMultiBulkLen(c,2); + addReplyArrayLen(c,2); addReplyHumanLongDouble(c,xy[0]); addReplyHumanLongDouble(c,xy[1]); } @@ -797,7 +805,7 @@ void geodistCommand(client *c) { /* Look up the requested zset */ robj *zobj = NULL; - if ((zobj = lookupKeyReadOrReply(c, c->argv[1], shared.nullbulk)) + if ((zobj = lookupKeyReadOrReply(c, c->argv[1], shared.null[c->resp])) == NULL || checkType(c, zobj, OBJ_ZSET)) return; /* Get the scores. We need both otherwise NULL is returned. */ @@ -805,13 +813,13 @@ void geodistCommand(client *c) { if (zsetScore(zobj, c->argv[2]->ptr, &score1) == C_ERR || zsetScore(zobj, c->argv[3]->ptr, &score2) == C_ERR) { - addReply(c,shared.nullbulk); + addReplyNull(c); return; } /* Decode & compute the distance. */ if (!decodeGeohash(score1,xyxy) || !decodeGeohash(score2,xyxy+2)) - addReply(c,shared.nullbulk); + addReplyNull(c); else addReplyDoubleDistance(c, geohashGetDistance(xyxy[0],xyxy[1],xyxy[2],xyxy[3]) / to_meter); diff --git a/redis.submodule/src/geohash.c b/redis.submodule/src/geohash.c index db5ae02..de9620b 100644 --- a/redis.submodule/src/geohash.c +++ b/redis.submodule/src/geohash.c @@ -206,7 +206,11 @@ int geohashDecodeWGS84(const GeoHashBits hash, GeoHashArea *area) { int geohashDecodeAreaToLongLat(const GeoHashArea *area, double *xy) { if (!xy) return 0; xy[0] = (area->longitude.min + area->longitude.max) / 2; + if (xy[0] > GEO_LONG_MAX) xy[0] = GEO_LONG_MAX; + if (xy[0] < GEO_LONG_MIN) xy[0] = GEO_LONG_MIN; xy[1] = (area->latitude.min + area->latitude.max) / 2; + if (xy[1] > GEO_LAT_MAX) xy[1] = GEO_LAT_MAX; + if (xy[1] < GEO_LAT_MIN) xy[1] = GEO_LAT_MIN; return 1; } diff --git a/redis.submodule/src/gopher.c b/redis.submodule/src/gopher.c new file mode 100644 index 0000000..38e44f7 --- /dev/null +++ b/redis.submodule/src/gopher.c @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2019, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "server.h" + +/* Emit an item in Gopher directory listing format: + * + * If descr or selector are NULL, then the "(NULL)" string is used instead. */ +void addReplyGopherItem(client *c, const char *type, const char *descr, + const char *selector, const char *hostname, int port) +{ + sds item = sdscatfmt(sdsempty(),"%s%s\t%s\t%s\t%i\r\n", + type, descr, + selector ? selector : "(NULL)", + hostname ? hostname : "(NULL)", + port); + addReplyProto(c,item,sdslen(item)); + sdsfree(item); +} + +/* This is called by processInputBuffer() when an inline request is processed + * with Gopher mode enabled, and the request happens to have zero or just one + * argument. In such case we get the relevant key and reply using the Gopher + * protocol. */ +void processGopherRequest(client *c) { + robj *keyname = c->argc == 0 ? createStringObject("/",1) : c->argv[0]; + robj *o = lookupKeyRead(c->db,keyname); + + /* If there is no such key, return with a Gopher error. */ + if (o == NULL || o->type != OBJ_STRING) { + char *errstr; + if (o == NULL) + errstr = "Error: no content at the specified key"; + else + errstr = "Error: selected key type is invalid " + "for Gopher output"; + addReplyGopherItem(c,"i",errstr,NULL,NULL,0); + addReplyGopherItem(c,"i","Redis Gopher server",NULL,NULL,0); + } else { + addReply(c,o); + } + + /* Cleanup, also make sure to emit the final ".CRLF" line. Note that + * the connection will be closed immediately after this because the client + * will be flagged with CLIENT_CLOSE_AFTER_REPLY, in accordance with the + * Gopher protocol. */ + if (c->argc == 0) decrRefCount(keyname); + + /* Note that in theory we should terminate the Gopher request with + * "." (called Lastline in the RFC) like that: + * + * addReplyProto(c,".\r\n",3); + * + * However after examining the current clients landscape, it's probably + * going to do more harm than good for several reasons: + * + * 1. Clients should not have any issue with missing . as for + * specification, and in the real world indeed certain servers + * implementations never used to send the terminator. + * + * 2. Redis does not know if it's serving a text file or a binary file: + * at the same time clients will not remove the "." bytes at + * tne end when downloading a binary file from the server, so adding + * the "Lastline" terminator without knowing the content is just + * dangerous. + * + * 3. The utility gopher2redis.rb that we provide for Redis, and any + * other similar tool you may use as Gopher authoring system for + * Redis, can just add the "Lastline" when needed. + */ +} diff --git a/redis.submodule/src/help.h b/redis.submodule/src/help.h index 184d767..6d3eb33 100644 --- a/redis.submodule/src/help.h +++ b/redis.submodule/src/help.h @@ -28,6 +28,56 @@ struct commandHelp { int group; char *since; } commandHelp[] = { + { "ACL CAT", + "[categoryname]", + "List the ACL categories or the commands inside a category", + 9, + "6.0.0" }, + { "ACL DELUSER", + "username [username ...]", + "Remove the specified ACL users and the associated rules", + 9, + "6.0.0" }, + { "ACL GENPASS", + "[bits]", + "Generate a pseudorandom secure password to use for ACL users", + 9, + "6.0.0" }, + { "ACL LIST", + "-", + "List the current ACL rules in ACL config file format", + 9, + "6.0.0" }, + { "ACL LOAD", + "-", + "Reload the ACLs from the configured ACL file", + 9, + "6.0.0" }, + { "ACL LOG", + "[count or RESET]", + "List latest events denied because of ACLs in place", + 9, + "6.0.0" }, + { "ACL SAVE", + "-", + "Save the current ACL rules in the configured ACL file", + 9, + "6.0.0" }, + { "ACL SETUSER", + "rule [rule ...]", + "Modify or create the rules for a specific ACL user", + 9, + "6.0.0" }, + { "ACL USERS", + "-", + "List the username of all the configured ACL rules", + 9, + "6.0.0" }, + { "ACL WHOAMI", + "-", + "Return the name of the user associated to the current connection", + 9, + "6.0.0" }, { "APPEND", "key value", "Append a value to a key", @@ -44,7 +94,7 @@ struct commandHelp { 9, "1.0.0" }, { "BGSAVE", - "-", + "[SCHEDULE]", "Asynchronously save the dataset to disk", 9, "1.0.0" }, @@ -80,7 +130,7 @@ struct commandHelp { "2.0.0" }, { "BRPOPLPUSH", "source destination timeout", - "Pop a value from a list, push it to another list and return it; or block until one is available", + "Pop an element from a list, push it to another list and return it; or block until one is available", 2, "2.2.0" }, { "BZPOPMAX", @@ -93,51 +143,71 @@ struct commandHelp { "Remove and return the member with the lowest score from one or more sorted sets, or block until one is available", 4, "5.0.0" }, + { "CLIENT CACHING", + "YES|NO", + "Instruct the server about tracking or not keys in the next request", + 8, + "6.0.0" }, { "CLIENT GETNAME", "-", "Get the current connection name", - 9, + 8, "2.6.9" }, + { "CLIENT GETREDIR", + "-", + "Get tracking notifications redirection client ID if any", + 8, + "6.0.0" }, { "CLIENT ID", "-", "Returns the client ID for the current connection", - 9, + 8, "5.0.0" }, { "CLIENT KILL", "[ip:port] [ID client-id] [TYPE normal|master|slave|pubsub] [ADDR ip:port] [SKIPME yes/no]", "Kill the connection of a client", - 9, + 8, "2.4.0" }, { "CLIENT LIST", - "-", + "[TYPE normal|master|replica|pubsub]", "Get the list of client connections", - 9, + 8, "2.4.0" }, { "CLIENT PAUSE", "timeout", "Stop processing commands from clients for some time", - 9, + 8, "2.9.50" }, { "CLIENT REPLY", "ON|OFF|SKIP", "Instruct the server whether to reply to commands", - 9, + 8, "3.2" }, { "CLIENT SETNAME", "connection-name", "Set the current connection name", - 9, + 8, "2.6.9" }, + { "CLIENT TRACKING", + "ON|OFF [REDIRECT client-id] [PREFIX prefix] [BCAST] [OPTIN] [OPTOUT] [NOLOOP]", + "Enable or disable server assisted client side caching support", + 8, + "6.0.0" }, { "CLIENT UNBLOCK", "client-id [TIMEOUT|ERROR]", "Unblock a client blocked in a blocking command from a different connection", - 9, + 8, "5.0.0" }, { "CLUSTER ADDSLOTS", "slot [slot ...]", "Assign new hash slots to receiving node", 12, "3.0.0" }, + { "CLUSTER BUMPEPOCH", + "-", + "Advance the cluster config epoch", + 12, + "3.0.0" }, { "CLUSTER COUNT-FAILURE-REPORTS", "node-id", "Return the number of failure reports active for a given node", @@ -158,6 +228,11 @@ struct commandHelp { "Forces a replica to perform a manual failover of its master.", 12, "3.0.0" }, + { "CLUSTER FLUSHSLOTS", + "-", + "Delete a node's own slots information", + 12, + "3.0.0" }, { "CLUSTER FORGET", "node-id", "Remove a node from the nodes table", @@ -183,6 +258,11 @@ struct commandHelp { "Force a node cluster to handshake with another node", 12, "3.0.0" }, + { "CLUSTER MYID", + "-", + "Return the node id", + 12, + "3.0.0" }, { "CLUSTER NODES", "-", "Get Cluster config for the node", @@ -359,7 +439,7 @@ struct commandHelp { 13, "3.2.0" }, { "GEODIST", - "key member1 member2 [unit]", + "key member1 member2 [m|km|ft|mi]", "Returns the distance between two members of a geospatial index", 13, "3.2.0" }, @@ -408,6 +488,11 @@ struct commandHelp { "Delete one or more hash fields", 5, "2.0.0" }, + { "HELLO", + "protover [AUTH username password] [SETNAME clientname]", + "switch Redis protocol", + 8, + "6.0.0" }, { "HEXISTS", "key field", "Determine if a hash field exists", @@ -459,7 +544,7 @@ struct commandHelp { 5, "2.8.0" }, { "HSET", - "key field value", + "key field value [field value ...]", "Set the string value of a hash field", 5, "2.0.0" }, @@ -508,13 +593,43 @@ struct commandHelp { "Get the UNIX time stamp of the last successful save to disk", 9, "1.0.0" }, + { "LATENCY DOCTOR", + "-", + "Return a human readable latency analysis report.", + 9, + "2.8.13" }, + { "LATENCY GRAPH", + "event", + "Return a latency graph for the event.", + 9, + "2.8.13" }, + { "LATENCY HELP", + "-", + "Show helpful text about the different subcommands.", + 9, + "2.8.13" }, + { "LATENCY HISTORY", + "event", + "Return timestamp-latency samples for the event.", + 9, + "2.8.13" }, + { "LATENCY LATEST", + "-", + "Return the latest latency samples for all events.", + 9, + "2.8.13" }, + { "LATENCY RESET", + "[event]", + "Reset latency data for one or more events.", + 9, + "2.8.13" }, { "LINDEX", "key index", "Get an element from a list by its index", 2, "1.0.0" }, { "LINSERT", - "key BEFORE|AFTER pivot value", + "key BEFORE|AFTER pivot element", "Insert an element before or after another element in a list", 2, "2.2.0" }, @@ -523,19 +638,24 @@ struct commandHelp { "Get the length of a list", 2, "1.0.0" }, + { "LOLWUT", + "[VERSION version]", + "Display some computer art and the Redis version", + 9, + "5.0.0" }, { "LPOP", "key", "Remove and get the first element in a list", 2, "1.0.0" }, { "LPUSH", - "key value [value ...]", - "Prepend one or multiple values to a list", + "key element [element ...]", + "Prepend one or multiple elements to a list", 2, "1.0.0" }, { "LPUSHX", - "key value", - "Prepend a value to a list, only if the list exists", + "key element [element ...]", + "Prepend an element to a list, only if the list exists", 2, "2.2.0" }, { "LRANGE", @@ -544,12 +664,12 @@ struct commandHelp { 2, "1.0.0" }, { "LREM", - "key count value", + "key count element", "Remove elements from a list", 2, "1.0.0" }, { "LSET", - "key index value", + "key index element", "Set the value of an element in a list by its index", 2, "1.0.0" }, @@ -594,10 +714,25 @@ struct commandHelp { 1, "1.0.0" }, { "MIGRATE", - "host port key|"" destination-db timeout [COPY] [REPLACE] [KEYS key]", + "host port key|"" destination-db timeout [COPY] [REPLACE] [AUTH password] [KEYS key]", "Atomically transfer a key from a Redis instance to another one.", 0, "2.6.0" }, + { "MODULE LIST", + "-", + "List all modules loaded by the server", + 9, + "4.0.0" }, + { "MODULE LOAD", + "path [arg]", + "Load a module", + 9, + "4.0.0" }, + { "MODULE UNLOAD", + "name", + "Unload a module", + 9, + "4.0.0" }, { "MONITOR", "-", "Listen for all requests received by the server in real time", @@ -673,6 +808,11 @@ struct commandHelp { "Listen for messages published to channels matching the given patterns", 6, "2.0.0" }, + { "PSYNC", + "replicationid offset", + "Internal command used for replication", + 9, + "2.8.0" }, { "PTTL", "key", "Get the time to live for a key in milliseconds", @@ -729,7 +869,7 @@ struct commandHelp { 9, "5.0.0" }, { "RESTORE", - "key ttl serialized-value [REPLACE]", + "key ttl serialized-value [REPLACE] [ABSTTL] [IDLETIME seconds] [FREQ frequency]", "Create a key using the provided serialized value, previously obtained using DUMP.", 0, "2.6.0" }, @@ -749,13 +889,13 @@ struct commandHelp { 2, "1.2.0" }, { "RPUSH", - "key value [value ...]", - "Append one or multiple values to a list", + "key element [element ...]", + "Append one or multiple elements to a list", 2, "1.0.0" }, { "RPUSHX", - "key value", - "Append a value to a list, only if the list exists", + "key element [element ...]", + "Append an element to a list, only if the list exists", 2, "2.2.0" }, { "SADD", @@ -769,7 +909,7 @@ struct commandHelp { 9, "1.0.0" }, { "SCAN", - "cursor [MATCH pattern] [COUNT count]", + "cursor [MATCH pattern] [COUNT count] [TYPE type]", "Incrementally iterate the keys space", 0, "2.8.0" }, @@ -819,7 +959,7 @@ struct commandHelp { 8, "1.0.0" }, { "SET", - "key value [expiration EX seconds|PX milliseconds] [NX|XX]", + "key value [EX seconds|PX milliseconds] [NX|XX] [KEEPTTL]", "Set the string value of a key", 1, "1.0.0" }, @@ -908,6 +1048,11 @@ struct commandHelp { "Incrementally iterate Set elements", 3, "2.8.0" }, + { "STRALGO", + "LCS algo-specific-argument [algo-specific-argument ...]", + "Run algorithms (currently LCS) against strings", + 1, + "6.0.0" }, { "STRLEN", "key", "Get the length of the value stored in a key", @@ -929,9 +1074,9 @@ struct commandHelp { 3, "1.0.0" }, { "SWAPDB", - "index index", + "index1 index2", "Swaps two Redis databases", - 8, + 9, "4.0.0" }, { "SYNC", "-", @@ -989,7 +1134,7 @@ struct commandHelp { 14, "5.0.0" }, { "XADD", - "key ID field string [field string ...]", + "key ID field value [field value ...]", "Appends a new entry to a stream", 14, "5.0.0" }, @@ -1004,7 +1149,7 @@ struct commandHelp { 14, "5.0.0" }, { "XGROUP", - "[CREATE key groupname id-or-$] [SETID key id-or-$] [DESTROY key groupname] [DELCONSUMER key groupname consumername]", + "[CREATE key groupname id-or-$] [SETID key groupname id-or-$] [DESTROY key groupname] [DELCONSUMER key groupname consumername]", "Create, destroy, and manage consumer groups.", 14, "5.0.0" }, @@ -1029,12 +1174,12 @@ struct commandHelp { 14, "5.0.0" }, { "XREAD", - "[COUNT count] [BLOCK milliseconds] STREAMS key [key ...] ID [ID ...]", + "[COUNT count] [BLOCK milliseconds] STREAMS key [key ...] id [id ...]", "Return never seen elements in multiple streams, with IDs greater than the ones reported by the caller for each stream. Can block.", 14, "5.0.0" }, { "XREADGROUP", - "GROUP group consumer [COUNT count] [BLOCK milliseconds] STREAMS key [key ...] ID [ID ...]", + "GROUP group consumer [COUNT count] [BLOCK milliseconds] [NOACK] STREAMS key [key ...] ID [ID ...]", "Return new entries from a stream using a consumer group, or access the history of the pending entries for a given consumer. Can block.", 14, "5.0.0" }, diff --git a/redis.submodule/src/hyperloglog.c b/redis.submodule/src/hyperloglog.c index a2c789d..721f492 100644 --- a/redis.submodule/src/hyperloglog.c +++ b/redis.submodule/src/hyperloglog.c @@ -1209,7 +1209,7 @@ void pfaddCommand(client *c) { } hdr = o->ptr; if (updated) { - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"pfadd",c->argv[1],c->db->id); server.dirty++; HLL_INVALIDATE_CACHE(hdr); @@ -1242,7 +1242,7 @@ void pfcountCommand(client *c) { if (o == NULL) continue; /* Assume empty HLL for non existing var.*/ if (isHLLObjectOrReply(c,o) != C_OK) return; - /* Merge with this HLL with our 'max' HHL by setting max[i] + /* Merge with this HLL with our 'max' HLL by setting max[i] * to MAX(max[i],hll[i]). */ if (hllMerge(registers,o) == C_ERR) { addReplySds(c,sdsnew(invalid_hll_err)); @@ -1300,7 +1300,7 @@ void pfcountCommand(client *c) { * data structure is not modified, since the cached value * may be modified and given that the HLL is a Redis string * we need to propagate the change. */ - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); server.dirty++; } addReplyLongLong(c,card); @@ -1329,7 +1329,7 @@ void pfmergeCommand(client *c) { hdr = o->ptr; if (hdr->encoding == HLL_DENSE) use_dense = 1; - /* Merge with this HLL with our 'max' HHL by setting max[i] + /* Merge with this HLL with our 'max' HLL by setting max[i] * to MAX(max[i],hll[i]). */ if (hllMerge(max,o) == C_ERR) { addReplySds(c,sdsnew(invalid_hll_err)); @@ -1373,7 +1373,7 @@ void pfmergeCommand(client *c) { last hllSparseSet() call. */ HLL_INVALIDATE_CACHE(hdr); - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); /* We generate a PFADD event for PFMERGE for semantical simplicity * since in theory this is a mass-add of elements. */ notifyKeyspaceEvent(NOTIFY_STRING,"pfadd",c->argv[1],c->db->id); @@ -1519,7 +1519,7 @@ void pfdebugCommand(client *c) { } hdr = o->ptr; - addReplyMultiBulkLen(c,HLL_REGISTERS); + addReplyArrayLen(c,HLL_REGISTERS); for (j = 0; j < HLL_REGISTERS; j++) { uint8_t val; @@ -1535,6 +1535,7 @@ void pfdebugCommand(client *c) { sds decoded = sdsempty(); if (hdr->encoding != HLL_SPARSE) { + sdsfree(decoded); addReplyError(c,"HLL encoding is not sparse"); return; } diff --git a/redis.submodule/src/latency.c b/redis.submodule/src/latency.c index 97e6a70..9a291ac 100644 --- a/redis.submodule/src/latency.c +++ b/redis.submodule/src/latency.c @@ -85,7 +85,7 @@ int THPGetAnonHugePagesSize(void) { /* ---------------------------- Latency API --------------------------------- */ /* Latency monitor initialization. We just need to create the dictionary - * of time series, each time serie is craeted on demand in order to avoid + * of time series, each time serie is created on demand in order to avoid * having a fixed list to maintain. */ void latencyMonitorInit(void) { server.latency_events = dictCreate(&latencyTimeSeriesDictType,NULL); @@ -95,7 +95,7 @@ void latencyMonitorInit(void) { * This function is usually called via latencyAddSampleIfNeeded(), that * is a macro that only adds the sample if the latency is higher than * server.latency_monitor_threshold. */ -void latencyAddSample(char *event, mstime_t latency) { +void latencyAddSample(const char *event, mstime_t latency) { struct latencyTimeSeries *ts = dictFetchValue(server.latency_events,event); time_t now = time(NULL); int prev; @@ -476,19 +476,19 @@ sds createLatencyReport(void) { /* latencyCommand() helper to produce a time-delay reply for all the samples * in memory for the specified time series. */ void latencyCommandReplyWithSamples(client *c, struct latencyTimeSeries *ts) { - void *replylen = addDeferredMultiBulkLength(c); + void *replylen = addReplyDeferredLen(c); int samples = 0, j; for (j = 0; j < LATENCY_TS_LEN; j++) { int i = (ts->idx + j) % LATENCY_TS_LEN; if (ts->samples[i].time == 0) continue; - addReplyMultiBulkLen(c,2); + addReplyArrayLen(c,2); addReplyLongLong(c,ts->samples[i].time); addReplyLongLong(c,ts->samples[i].latency); samples++; } - setDeferredMultiBulkLength(c,replylen,samples); + setDeferredArrayLen(c,replylen,samples); } /* latencyCommand() helper to produce the reply for the LATEST subcommand, @@ -497,14 +497,14 @@ void latencyCommandReplyWithLatestEvents(client *c) { dictIterator *di; dictEntry *de; - addReplyMultiBulkLen(c,dictSize(server.latency_events)); + addReplyArrayLen(c,dictSize(server.latency_events)); di = dictGetIterator(server.latency_events); while((de = dictNext(di)) != NULL) { char *event = dictGetKey(de); struct latencyTimeSeries *ts = dictGetVal(de); int last = (ts->idx + LATENCY_TS_LEN - 1) % LATENCY_TS_LEN; - addReplyMultiBulkLen(c,4); + addReplyArrayLen(c,4); addReplyBulkCString(c,event); addReplyLongLong(c,ts->samples[last].time); addReplyLongLong(c,ts->samples[last].latency); @@ -583,7 +583,7 @@ NULL /* LATENCY HISTORY */ ts = dictFetchValue(server.latency_events,c->argv[2]->ptr); if (ts == NULL) { - addReplyMultiBulkLen(c,0); + addReplyArrayLen(c,0); } else { latencyCommandReplyWithSamples(c,ts); } @@ -599,7 +599,7 @@ NULL event = dictGetKey(de); graph = latencyCommandGenSparkeline(event,ts); - addReplyBulkCString(c,graph); + addReplyVerbatim(c,graph,sdslen(graph),"txt"); sdsfree(graph); } else if (!strcasecmp(c->argv[1]->ptr,"latest") && c->argc == 2) { /* LATENCY LATEST */ @@ -608,7 +608,7 @@ NULL /* LATENCY DOCTOR */ sds report = createLatencyReport(); - addReplyBulkCBuffer(c,report,sdslen(report)); + addReplyVerbatim(c,report,sdslen(report),"txt"); sdsfree(report); } else if (!strcasecmp(c->argv[1]->ptr,"reset") && c->argc >= 2) { /* LATENCY RESET */ diff --git a/redis.submodule/src/latency.h b/redis.submodule/src/latency.h index 0fe26e0..76640cf 100644 --- a/redis.submodule/src/latency.h +++ b/redis.submodule/src/latency.h @@ -62,7 +62,7 @@ struct latencyStats { }; void latencyMonitorInit(void); -void latencyAddSample(char *event, mstime_t latency); +void latencyAddSample(const char *event, mstime_t latency); int THPIsEnabled(void); /* Latency monitoring macros. */ diff --git a/redis.submodule/src/lazyfree.c b/redis.submodule/src/lazyfree.c index 3d3159c..f01504e 100644 --- a/redis.submodule/src/lazyfree.c +++ b/redis.submodule/src/lazyfree.c @@ -83,7 +83,7 @@ int dbAsyncDelete(redisDb *db, robj *key) { * field to NULL in order to lazy free it later. */ if (de) { dictFreeUnlinkedEntry(db->dict,de); - if (server.cluster_enabled) slotToKeyDel(key); + if (server.cluster_enabled) slotToKeyDel(key->ptr); return 1; } else { return 0; diff --git a/redis.submodule/src/localtime.c b/redis.submodule/src/localtime.c index 3f59a33..e2ac81f 100644 --- a/redis.submodule/src/localtime.c +++ b/redis.submodule/src/localtime.c @@ -52,8 +52,8 @@ static int is_leap_year(time_t year) { if (year % 4) return 0; /* A year not divisible by 4 is not leap. */ else if (year % 100) return 1; /* If div by 4 and not 100 is surely leap. */ - else if (year % 400) return 0; /* If div by 100 *and* 400 is not leap. */ - else return 1; /* If div by 100 and not by 400 is leap. */ + else if (year % 400) return 0; /* If div by 100 *and* not by 400 is not leap. */ + else return 1; /* If div by 100 and 400 is leap. */ } void nolocks_localtime(struct tm *tmp, time_t t, time_t tz, int dst) { diff --git a/redis.submodule/src/lolwut.c b/redis.submodule/src/lolwut.c index 19cbcf6..0e1552b 100644 --- a/redis.submodule/src/lolwut.c +++ b/redis.submodule/src/lolwut.c @@ -34,8 +34,11 @@ */ #include "server.h" +#include "lolwut.h" +#include void lolwut5Command(client *c); +void lolwut6Command(client *c); /* The default target for LOLWUT if no matching version was found. * This is what unstable versions of Redis will display. */ @@ -43,14 +46,143 @@ void lolwutUnstableCommand(client *c) { sds rendered = sdsnew("Redis ver. "); rendered = sdscat(rendered,REDIS_VERSION); rendered = sdscatlen(rendered,"\n",1); - addReplyBulkSds(c,rendered); + addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); + sdsfree(rendered); } +/* LOLWUT [VERSION ] [... version specific arguments ...] */ void lolwutCommand(client *c) { char *v = REDIS_VERSION; - if ((v[0] == '5' && v[1] == '.') || + char verstr[64]; + + if (c->argc >= 3 && !strcasecmp(c->argv[1]->ptr,"version")) { + long ver; + if (getLongFromObjectOrReply(c,c->argv[2],&ver,NULL) != C_OK) return; + snprintf(verstr,sizeof(verstr),"%u.0.0",(unsigned int)ver); + v = verstr; + + /* Adjust argv/argc to filter the "VERSION ..." option, since the + * specific LOLWUT version implementations don't know about it + * and expect their arguments. */ + c->argv += 2; + c->argc -= 2; + } + + if ((v[0] == '5' && v[1] == '.' && v[2] != '9') || (v[0] == '4' && v[1] == '.' && v[2] == '9')) lolwut5Command(c); + else if ((v[0] == '6' && v[1] == '.' && v[2] != '9') || + (v[0] == '5' && v[1] == '.' && v[2] == '9')) + lolwut6Command(c); else lolwutUnstableCommand(c); + + /* Fix back argc/argv in case of VERSION argument. */ + if (v == verstr) { + c->argv -= 2; + c->argc += 2; + } +} + +/* ========================== LOLWUT Canvase =============================== + * Many LOWUT versions will likely print some computer art to the screen. + * This is the case with LOLWUT 5 and LOLWUT 6, so here there is a generic + * canvas implementation that can be reused. */ + +/* Allocate and return a new canvas of the specified size. */ +lwCanvas *lwCreateCanvas(int width, int height, int bgcolor) { + lwCanvas *canvas = zmalloc(sizeof(*canvas)); + canvas->width = width; + canvas->height = height; + canvas->pixels = zmalloc(width*height); + memset(canvas->pixels,bgcolor,width*height); + return canvas; +} + +/* Free the canvas created by lwCreateCanvas(). */ +void lwFreeCanvas(lwCanvas *canvas) { + zfree(canvas->pixels); + zfree(canvas); +} + +/* Set a pixel to the specified color. Color is 0 or 1, where zero means no + * dot will be displyed, and 1 means dot will be displayed. + * Coordinates are arranged so that left-top corner is 0,0. You can write + * out of the size of the canvas without issues. */ +void lwDrawPixel(lwCanvas *canvas, int x, int y, int color) { + if (x < 0 || x >= canvas->width || + y < 0 || y >= canvas->height) return; + canvas->pixels[x+y*canvas->width] = color; +} + +/* Return the value of the specified pixel on the canvas. */ +int lwGetPixel(lwCanvas *canvas, int x, int y) { + if (x < 0 || x >= canvas->width || + y < 0 || y >= canvas->height) return 0; + return canvas->pixels[x+y*canvas->width]; +} + +/* Draw a line from x1,y1 to x2,y2 using the Bresenham algorithm. */ +void lwDrawLine(lwCanvas *canvas, int x1, int y1, int x2, int y2, int color) { + int dx = abs(x2-x1); + int dy = abs(y2-y1); + int sx = (x1 < x2) ? 1 : -1; + int sy = (y1 < y2) ? 1 : -1; + int err = dx-dy, e2; + + while(1) { + lwDrawPixel(canvas,x1,y1,color); + if (x1 == x2 && y1 == y2) break; + e2 = err*2; + if (e2 > -dy) { + err -= dy; + x1 += sx; + } + if (e2 < dx) { + err += dx; + y1 += sy; + } + } +} + +/* Draw a square centered at the specified x,y coordinates, with the specified + * rotation angle and size. In order to write a rotated square, we use the + * trivial fact that the parametric equation: + * + * x = sin(k) + * y = cos(k) + * + * Describes a circle for values going from 0 to 2*PI. So basically if we start + * at 45 degrees, that is k = PI/4, with the first point, and then we find + * the other three points incrementing K by PI/2 (90 degrees), we'll have the + * points of the square. In order to rotate the square, we just start with + * k = PI/4 + rotation_angle, and we are done. + * + * Of course the vanilla equations above will describe the square inside a + * circle of radius 1, so in order to draw larger squares we'll have to + * multiply the obtained coordinates, and then translate them. However this + * is much simpler than implementing the abstract concept of 2D shape and then + * performing the rotation/translation transformation, so for LOLWUT it's + * a good approach. */ +void lwDrawSquare(lwCanvas *canvas, int x, int y, float size, float angle, int color) { + int px[4], py[4]; + + /* Adjust the desired size according to the fact that the square inscribed + * into a circle of radius 1 has the side of length SQRT(2). This way + * size becomes a simple multiplication factor we can use with our + * coordinates to magnify them. */ + size /= 1.4142135623; + size = round(size); + + /* Compute the four points. */ + float k = M_PI/4 + angle; + for (int j = 0; j < 4; j++) { + px[j] = round(sin(k) * size + x); + py[j] = round(cos(k) * size + y); + k += M_PI/2; + } + + /* Draw the square. */ + for (int j = 0; j < 4; j++) + lwDrawLine(canvas,px[j],py[j],px[(j+1)%4],py[(j+1)%4],color); } diff --git a/redis.submodule/src/lolwut.h b/redis.submodule/src/lolwut.h new file mode 100644 index 0000000..682d005 --- /dev/null +++ b/redis.submodule/src/lolwut.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2018-2019, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* This structure represents our canvas. Drawing functions will take a pointer + * to a canvas to write to it. Later the canvas can be rendered to a string + * suitable to be printed on the screen, using unicode Braille characters. */ + +/* This represents a very simple generic canvas in order to draw stuff. + * It's up to each LOLWUT versions to translate what they draw to the + * screen, depending on the result to accomplish. */ + +#ifndef __LOLWUT_H +#define __LOLWUT_H + +typedef struct lwCanvas { + int width; + int height; + char *pixels; +} lwCanvas; + +/* Drawing functions implemented inside lolwut.c. */ +lwCanvas *lwCreateCanvas(int width, int height, int bgcolor); +void lwFreeCanvas(lwCanvas *canvas); +void lwDrawPixel(lwCanvas *canvas, int x, int y, int color); +int lwGetPixel(lwCanvas *canvas, int x, int y); +void lwDrawLine(lwCanvas *canvas, int x1, int y1, int x2, int y2, int color); +void lwDrawSquare(lwCanvas *canvas, int x, int y, float size, float angle, int color); + +#endif diff --git a/redis.submodule/src/lolwut5.c b/redis.submodule/src/lolwut5.c index 8408b37..5a93488 100644 --- a/redis.submodule/src/lolwut5.c +++ b/redis.submodule/src/lolwut5.c @@ -34,17 +34,9 @@ */ #include "server.h" +#include "lolwut.h" #include -/* This structure represents our canvas. Drawing functions will take a pointer - * to a canvas to write to it. Later the canvas can be rendered to a string - * suitable to be printed on the screen, using unicode Braille characters. */ -typedef struct lwCanvas { - int width; - int height; - char *pixels; -} lwCanvas; - /* Translate a group of 8 pixels (2x4 vertical rectangle) to the corresponding * braille character. The byte should correspond to the pixels arranged as * follows, where 0 is the least significant bit, and 7 the most significant @@ -69,104 +61,6 @@ void lwTranslatePixelsGroup(int byte, char *output) { output[2] = 0x80 | (code & 0x3F); /* 10-xxxxxx */ } -/* Allocate and return a new canvas of the specified size. */ -lwCanvas *lwCreateCanvas(int width, int height) { - lwCanvas *canvas = zmalloc(sizeof(*canvas)); - canvas->width = width; - canvas->height = height; - canvas->pixels = zmalloc(width*height); - memset(canvas->pixels,0,width*height); - return canvas; -} - -/* Free the canvas created by lwCreateCanvas(). */ -void lwFreeCanvas(lwCanvas *canvas) { - zfree(canvas->pixels); - zfree(canvas); -} - -/* Set a pixel to the specified color. Color is 0 or 1, where zero means no - * dot will be displyed, and 1 means dot will be displayed. - * Coordinates are arranged so that left-top corner is 0,0. You can write - * out of the size of the canvas without issues. */ -void lwDrawPixel(lwCanvas *canvas, int x, int y, int color) { - if (x < 0 || x >= canvas->width || - y < 0 || y >= canvas->height) return; - canvas->pixels[x+y*canvas->width] = color; -} - -/* Return the value of the specified pixel on the canvas. */ -int lwGetPixel(lwCanvas *canvas, int x, int y) { - if (x < 0 || x >= canvas->width || - y < 0 || y >= canvas->height) return 0; - return canvas->pixels[x+y*canvas->width]; -} - -/* Draw a line from x1,y1 to x2,y2 using the Bresenham algorithm. */ -void lwDrawLine(lwCanvas *canvas, int x1, int y1, int x2, int y2, int color) { - int dx = abs(x2-x1); - int dy = abs(y2-y1); - int sx = (x1 < x2) ? 1 : -1; - int sy = (y1 < y2) ? 1 : -1; - int err = dx-dy, e2; - - while(1) { - lwDrawPixel(canvas,x1,y1,color); - if (x1 == x2 && y1 == y2) break; - e2 = err*2; - if (e2 > -dy) { - err -= dy; - x1 += sx; - } - if (e2 < dx) { - err += dx; - y1 += sy; - } - } -} - -/* Draw a square centered at the specified x,y coordinates, with the specified - * rotation angle and size. In order to write a rotated square, we use the - * trivial fact that the parametric equation: - * - * x = sin(k) - * y = cos(k) - * - * Describes a circle for values going from 0 to 2*PI. So basically if we start - * at 45 degrees, that is k = PI/4, with the first point, and then we find - * the other three points incrementing K by PI/2 (90 degrees), we'll have the - * points of the square. In order to rotate the square, we just start with - * k = PI/4 + rotation_angle, and we are done. - * - * Of course the vanilla equations above will describe the square inside a - * circle of radius 1, so in order to draw larger squares we'll have to - * multiply the obtained coordinates, and then translate them. However this - * is much simpler than implementing the abstract concept of 2D shape and then - * performing the rotation/translation transformation, so for LOLWUT it's - * a good approach. */ -void lwDrawSquare(lwCanvas *canvas, int x, int y, float size, float angle) { - int px[4], py[4]; - - /* Adjust the desired size according to the fact that the square inscribed - * into a circle of radius 1 has the side of length SQRT(2). This way - * size becomes a simple multiplication factor we can use with our - * coordinates to magnify them. */ - size /= 1.4142135623; - size = round(size); - - /* Compute the four points. */ - float k = M_PI/4 + angle; - for (int j = 0; j < 4; j++) { - px[j] = round(sin(k) * size + x); - py[j] = round(cos(k) * size + y); - k += M_PI/2; - } - - /* Draw the square. */ - for (int j = 0; j < 4; j++) - lwDrawLine(canvas,px[j],py[j],px[(j+1)%4],py[(j+1)%4],1); -} - /* Schotter, the output of LOLWUT of Redis 5, is a computer graphic art piece * generated by Georg Nees in the 60s. It explores the relationship between * caos and order. @@ -180,7 +74,7 @@ lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_ int padding = canvas_width > 4 ? 2 : 0; float square_side = (float)(canvas_width-padding*2) / squares_per_row; int canvas_height = square_side * squares_per_col + padding*2; - lwCanvas *canvas = lwCreateCanvas(canvas_width, canvas_height); + lwCanvas *canvas = lwCreateCanvas(canvas_width, canvas_height, 0); for (int y = 0; y < squares_per_col; y++) { for (int x = 0; x < squares_per_row; x++) { @@ -200,7 +94,7 @@ lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_ sx += r2*square_side/3; sy += r3*square_side/3; } - lwDrawSquare(canvas,sx,sy,square_side,angle); + lwDrawSquare(canvas,sx,sy,square_side,angle,1); } } @@ -212,7 +106,7 @@ lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_ * logical canvas. The actual returned string will require a terminal that is * width/2 large and height/4 tall in order to hold the whole image without * overflowing or scrolling, since each Barille character is 2x4. */ -sds lwRenderCanvas(lwCanvas *canvas) { +static sds renderCanvas(lwCanvas *canvas) { sds text = sdsempty(); for (int y = 0; y < canvas->height; y += 4) { for (int x = 0; x < canvas->width; x += 2) { @@ -272,11 +166,12 @@ void lolwut5Command(client *c) { /* Generate some computer art and reply. */ lwCanvas *canvas = lwDrawSchotter(cols,squares_per_row,squares_per_col); - sds rendered = lwRenderCanvas(canvas); + sds rendered = renderCanvas(canvas); rendered = sdscat(rendered, "\nGeorg Nees - schotter, plotter on paper, 1968. Redis ver. "); rendered = sdscat(rendered,REDIS_VERSION); rendered = sdscatlen(rendered,"\n",1); - addReplyBulkSds(c,rendered); + addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); + sdsfree(rendered); lwFreeCanvas(canvas); } diff --git a/redis.submodule/src/lolwut6.c b/redis.submodule/src/lolwut6.c new file mode 100644 index 0000000..471bf66 --- /dev/null +++ b/redis.submodule/src/lolwut6.c @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2019, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ---------------------------------------------------------------------------- + * + * This file implements the LOLWUT command. The command should do something + * fun and interesting, and should be replaced by a new implementation at + * each new version of Redis. + * + * Thanks to Michele Hiki Falcone for the original image that ispired + * the image, part of his game, Plaguemon. + * + * Thanks to the Shhh computer art collective for the help in tuning the + * output to have a better artistic effect. + */ + +#include "server.h" +#include "lolwut.h" + +/* Render the canvas using the four gray levels of the standard color + * terminal: they match very well to the grayscale display of the gameboy. */ +static sds renderCanvas(lwCanvas *canvas) { + sds text = sdsempty(); + for (int y = 0; y < canvas->height; y++) { + for (int x = 0; x < canvas->width; x++) { + int color = lwGetPixel(canvas,x,y); + char *ce; /* Color escape sequence. */ + + /* Note that we set both the foreground and background color. + * This way we are able to get a more consistent result among + * different terminals implementations. */ + switch(color) { + case 0: ce = "0;30;40m"; break; /* Black */ + case 1: ce = "0;90;100m"; break; /* Gray 1 */ + case 2: ce = "0;37;47m"; break; /* Gray 2 */ + case 3: ce = "0;97;107m"; break; /* White */ + default: ce = "0;30;40m"; break; /* Just for safety. */ + } + text = sdscatprintf(text,"\033[%s \033[0m",ce); + } + if (y != canvas->height-1) text = sdscatlen(text,"\n",1); + } + return text; +} + +/* Draw a skyscraper on the canvas, according to the parameters in the + * 'skyscraper' structure. Window colors are random and are always one + * of the two grays. */ +struct skyscraper { + int xoff; /* X offset. */ + int width; /* Pixels width. */ + int height; /* Pixels height. */ + int windows; /* Draw windows if true. */ + int color; /* Color of the skyscraper. */ +}; + +void generateSkyscraper(lwCanvas *canvas, struct skyscraper *si) { + int starty = canvas->height-1; + int endy = starty - si->height + 1; + for (int y = starty; y >= endy; y--) { + for (int x = si->xoff; x < si->xoff+si->width; x++) { + /* The roof is four pixels less wide. */ + if (y == endy && (x <= si->xoff+1 || x >= si->xoff+si->width-2)) + continue; + int color = si->color; + /* Alter the color if this is a place where we want to + * draw a window. We check that we are in the inner part of the + * skyscraper, so that windows are far from the borders. */ + if (si->windows && + x > si->xoff+1 && + x < si->xoff+si->width-2 && + y > endy+1 && + y < starty-1) + { + /* Calculate the x,y position relative to the start of + * the window area. */ + int relx = x - (si->xoff+1); + int rely = y - (endy+1); + + /* Note that we want the windows to be two pixels wide + * but just one pixel tall, because terminal "pixels" + * (characters) are not square. */ + if (relx/2 % 2 && rely % 2) { + do { + color = 1 + rand() % 2; + } while (color == si->color); + /* Except we want adjacent pixels creating the same + * window to be the same color. */ + if (relx % 2) color = lwGetPixel(canvas,x-1,y); + } + } + lwDrawPixel(canvas,x,y,color); + } + } +} + +/* Generate a skyline inspired by the parallax backgrounds of 8 bit games. */ +void generateSkyline(lwCanvas *canvas) { + struct skyscraper si; + + /* First draw the background skyscraper without windows, using the + * two different grays. We use two passes to make sure that the lighter + * ones are always in the background. */ + for (int color = 2; color >= 1; color--) { + si.color = color; + for (int offset = -10; offset < canvas->width;) { + offset += rand() % 8; + si.xoff = offset; + si.width = 10 + rand()%9; + if (color == 2) + si.height = canvas->height/2 + rand()%canvas->height/2; + else + si.height = canvas->height/2 + rand()%canvas->height/3; + si.windows = 0; + generateSkyscraper(canvas, &si); + if (color == 2) + offset += si.width/2; + else + offset += si.width+1; + } + } + + /* Now draw the foreground skyscraper with the windows. */ + si.color = 0; + for (int offset = -10; offset < canvas->width;) { + offset += rand() % 8; + si.xoff = offset; + si.width = 5 + rand()%14; + if (si.width % 4) si.width += (si.width % 3); + si.height = canvas->height/3 + rand()%canvas->height/2; + si.windows = 1; + generateSkyscraper(canvas, &si); + offset += si.width+5; + } +} + +/* The LOLWUT 6 command: + * + * LOLWUT [columns] [rows] + * + * By default the command uses 80 columns, 40 squares per row + * per column. + */ +void lolwut6Command(client *c) { + long cols = 80; + long rows = 20; + + /* Parse the optional arguments if any. */ + if (c->argc > 1 && + getLongFromObjectOrReply(c,c->argv[1],&cols,NULL) != C_OK) + return; + + if (c->argc > 2 && + getLongFromObjectOrReply(c,c->argv[2],&rows,NULL) != C_OK) + return; + + /* Limits. We want LOLWUT to be always reasonably fast and cheap to execute + * so we have maximum number of columns, rows, and output resulution. */ + if (cols < 1) cols = 1; + if (cols > 1000) cols = 1000; + if (rows < 1) rows = 1; + if (rows > 1000) rows = 1000; + + /* Generate the city skyline and reply. */ + lwCanvas *canvas = lwCreateCanvas(cols,rows,3); + generateSkyline(canvas); + sds rendered = renderCanvas(canvas); + rendered = sdscat(rendered, + "\nDedicated to the 8 bit game developers of past and present.\n" + "Original 8 bit image from Plaguemon by hikikomori. Redis ver. "); + rendered = sdscat(rendered,REDIS_VERSION); + rendered = sdscatlen(rendered,"\n",1); + addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); + sdsfree(rendered); + lwFreeCanvas(canvas); +} diff --git a/redis.submodule/src/mkreleasehdr.sh b/redis.submodule/src/mkreleasehdr.sh index e6d558b..236c26c 100755 --- a/redis.submodule/src/mkreleasehdr.sh +++ b/redis.submodule/src/mkreleasehdr.sh @@ -3,7 +3,7 @@ GIT_SHA1=`(git show-ref --head --hash=8 2> /dev/null || echo 00000000) | head -n GIT_DIRTY=`git diff --no-ext-diff 2> /dev/null | wc -l` BUILD_ID=`uname -n`"-"`date +%s` if [ -n "$SOURCE_DATE_EPOCH" ]; then - BUILD_ID=$(date -u -d "@$SOURCE_DATE_EPOCH" +%s 2>/dev/null || date -u -r "$SOURCE_DATE_EPOCH" +%s 2>/dev/null || date -u %s) + BUILD_ID=$(date -u -d "@$SOURCE_DATE_EPOCH" +%s 2>/dev/null || date -u -r "$SOURCE_DATE_EPOCH" +%s 2>/dev/null || date -u +%s) fi test -f release.h || touch release.h (cat release.h | grep SHA1 | grep $GIT_SHA1) && \ diff --git a/redis.submodule/src/module.c b/redis.submodule/src/module.c index d55bb4e..e3a338d 100644 --- a/redis.submodule/src/module.c +++ b/redis.submodule/src/module.c @@ -31,9 +31,8 @@ #include "cluster.h" #include "rdb.h" #include - -#define REDISMODULE_CORE 1 -#include "redismodule.h" +#include +#include /* -------------------------------------------------------------------------- * Private data structures used by the modules system. Those are data @@ -41,6 +40,17 @@ * pointers that have an API the module can call with them) * -------------------------------------------------------------------------- */ +typedef struct RedisModuleInfoCtx { + struct RedisModule *module; + const char *requested_section; + sds info; /* info string we collected so far */ + int sections; /* number of sections we collected so far */ + int in_section; /* indication if we're in an active section or not */ + int in_dict_field; /* indication that we're curreintly appending to a dict */ +} RedisModuleInfoCtx; + +typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_report); + /* This structure represents a module inside the system. */ struct RedisModule { void *handle; /* Module dlopen() handle. */ @@ -52,6 +62,10 @@ struct RedisModule { list *using; /* List of modules we use some APIs of. */ list *filters; /* List of filters the module has registered. */ int in_call; /* RM_Call() nesting level */ + int in_hook; /* Hooks callback nesting level for this module (0 or 1). */ + int options; /* Module options and capabilities. */ + int blocked_clients; /* Count of RedisModuleBlockedClient in this module. */ + RedisModuleInfoFunc info_cb; /* Callback for module to add INFO fields. */ }; typedef struct RedisModule RedisModule; @@ -80,6 +94,7 @@ struct AutoMemEntry { #define REDISMODULE_AM_REPLY 2 #define REDISMODULE_AM_FREED 3 /* Explicitly freed by user already. */ #define REDISMODULE_AM_DICT 4 +#define REDISMODULE_AM_INFO 5 /* The pool allocator block. Redis Modules can allocate memory via this special * allocator that will automatically release it all once the callback returns. @@ -127,16 +142,23 @@ struct RedisModuleCtx { void **postponed_arrays; /* To set with RM_ReplySetArrayLength(). */ int postponed_arrays_count; /* Number of entries in postponed_arrays. */ void *blocked_privdata; /* Privdata set when unblocking a client. */ + RedisModuleString *blocked_ready_key; /* Key ready when the reply callback + gets called for clients blocked + on keys. */ /* Used if there is the REDISMODULE_CTX_KEYS_POS_REQUEST flag set. */ int *keys_pos; int keys_count; struct RedisModulePoolAllocBlock *pa_head; + redisOpArray saved_oparray; /* When propagating commands in a callback + we reallocate the "also propagate" op + array. Here we save the old one to + restore it later. */ }; typedef struct RedisModuleCtx RedisModuleCtx; -#define REDISMODULE_CTX_INIT {(void*)(unsigned long)&RM_GetApi, NULL, NULL, NULL, NULL, 0, 0, 0, NULL, 0, NULL, NULL, 0, NULL} +#define REDISMODULE_CTX_INIT {(void*)(unsigned long)&RM_GetApi, NULL, NULL, NULL, NULL, 0, 0, 0, NULL, 0, NULL, NULL, NULL, 0, NULL, {0}} #define REDISMODULE_CTX_MULTI_EMITTED (1<<0) #define REDISMODULE_CTX_AUTO_MEMORY (1<<1) #define REDISMODULE_CTX_KEYS_POS_REQUEST (1<<2) @@ -144,6 +166,7 @@ typedef struct RedisModuleCtx RedisModuleCtx; #define REDISMODULE_CTX_BLOCKED_TIMEOUT (1<<4) #define REDISMODULE_CTX_THREAD_SAFE (1<<5) #define REDISMODULE_CTX_BLOCKED_DISCONNECTED (1<<6) +#define REDISMODULE_CTX_MODULE_COMMAND_CALL (1<<7) /* This represents a Redis key opened with RM_OpenKey(). */ struct RedisModuleKey { @@ -227,6 +250,8 @@ typedef struct RedisModuleBlockedClient { client *reply_client; /* Fake client used to accumulate replies in thread safe contexts. */ int dbid; /* Database number selected by the original client. */ + int blocked_on_keys; /* If blocked via RM_BlockClientOnKeys(). */ + int unblocked; /* Already on the moduleUnblocked list. */ } RedisModuleBlockedClient; static pthread_mutex_t moduleUnblockedClientsMutex = PTHREAD_MUTEX_INITIALIZER; @@ -292,6 +317,59 @@ typedef struct RedisModuleCommandFilter { /* Registered filters */ static list *moduleCommandFilters; +typedef void (*RedisModuleForkDoneHandler) (int exitcode, int bysignal, void *user_data); + +static struct RedisModuleForkInfo { + RedisModuleForkDoneHandler done_handler; + void* done_handler_user_data; +} moduleForkInfo = {0}; + +typedef struct RedisModuleServerInfoData { + rax *rax; /* parsed info data. */ +} RedisModuleServerInfoData; + +/* Flags for moduleCreateArgvFromUserFormat(). */ +#define REDISMODULE_ARGV_REPLICATE (1<<0) +#define REDISMODULE_ARGV_NO_AOF (1<<1) +#define REDISMODULE_ARGV_NO_REPLICAS (1<<2) + +/* Determine whether Redis should signalModifiedKey implicitly. + * In case 'ctx' has no 'module' member (and therefore no module->options), + * we assume default behavior, that is, Redis signals. + * (see RM_GetThreadSafeContext) */ +#define SHOULD_SIGNAL_MODIFIED_KEYS(ctx) \ + ctx->module? !(ctx->module->options & REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED) : 1 + +/* Server events hooks data structures and defines: this modules API + * allow modules to subscribe to certain events in Redis, such as + * the start and end of an RDB or AOF save, the change of role in replication, + * and similar other events. */ + +typedef struct RedisModuleEventListener { + RedisModule *module; + RedisModuleEvent event; + RedisModuleEventCallback callback; +} RedisModuleEventListener; + +list *RedisModule_EventListeners; /* Global list of all the active events. */ +unsigned long long ModulesInHooks = 0; /* Total number of modules in hooks + callbacks right now. */ + +/* Data structures related to the redis module users */ + +/* This callback type is called by moduleNotifyUserChanged() every time + * a user authenticated via the module API is associated with a different + * user or gets disconnected. */ +typedef void (*RedisModuleUserChangedFunc) (uint64_t client_id, void *privdata); + +/* This is the object returned by RM_CreateModuleUser(). The module API is + * able to create users, set ACLs to such users, and later authenticate + * clients using such newly created users. */ +typedef struct RedisModuleUser { + user *user; /* Reference to the real redis user */ +} RedisModuleUser; + + /* -------------------------------------------------------------------------- * Prototypes * -------------------------------------------------------------------------- */ @@ -304,6 +382,7 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx); void RM_ZsetRangeStop(RedisModuleKey *kp); static void zsetKeyReset(RedisModuleKey *key); void RM_FreeDict(RedisModuleCtx *ctx, RedisModuleDict *d); +void RM_FreeServerInfo(RedisModuleCtx *ctx, RedisModuleServerInfoData *data); /* -------------------------------------------------------------------------- * Heap allocation raw functions @@ -461,7 +540,8 @@ int moduleDelKeyIfEmpty(RedisModuleKey *key) { case OBJ_LIST: isempty = listTypeLength(o) == 0; break; case OBJ_SET: isempty = setTypeSize(o) == 0; break; case OBJ_ZSET: isempty = zsetLength(o) == 0; break; - case OBJ_HASH : isempty = hashTypeLength(o) == 0; break; + case OBJ_HASH: isempty = hashTypeLength(o) == 0; break; + case OBJ_STREAM: isempty = streamLength(o) == 0; break; default: isempty = 0; } @@ -497,8 +577,44 @@ int RM_GetApi(const char *funcname, void **targetPtrPtr) { return REDISMODULE_OK; } +/* Helper function for when a command callback is called, in order to handle + * details needed to correctly replicate commands. */ +void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) { + client *c = ctx->client; + + /* We don't need to do anything here if the context was never used + * in order to propagate commands. */ + if (!(ctx->flags & REDISMODULE_CTX_MULTI_EMITTED)) return; + + if (c->flags & CLIENT_LUA) return; + + /* Handle the replication of the final EXEC, since whatever a command + * emits is always wrapped around MULTI/EXEC. */ + alsoPropagate(server.execCommand,c->db->id,&shared.exec,1, + PROPAGATE_AOF|PROPAGATE_REPL); + + /* If this is not a module command context (but is instead a simple + * callback context), we have to handle directly the "also propagate" + * array and emit it. In a module command call this will be handled + * directly by call(). */ + if (!(ctx->flags & REDISMODULE_CTX_MODULE_COMMAND_CALL) && + server.also_propagate.numops) + { + for (int j = 0; j < server.also_propagate.numops; j++) { + redisOp *rop = &server.also_propagate.ops[j]; + int target = rop->target; + if (target) + propagate(rop->cmd,rop->dbid,rop->argv,rop->argc,target); + } + redisOpArrayFree(&server.also_propagate); + /* Restore the previous oparray in case of nexted use of the API. */ + server.also_propagate = ctx->saved_oparray; + } +} + /* Free the context after the user function was called. */ void moduleFreeContext(RedisModuleCtx *ctx) { + moduleHandlePropagationAfterCommandCallback(ctx); autoMemoryCollect(ctx); poolAllocRelease(ctx); if (ctx->postponed_arrays) { @@ -514,34 +630,16 @@ void moduleFreeContext(RedisModuleCtx *ctx) { if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) freeClient(ctx->client); } -/* Helper function for when a command callback is called, in order to handle - * details needed to correctly replicate commands. */ -void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) { - client *c = ctx->client; - - if (c->flags & CLIENT_LUA) return; - - /* Handle the replication of the final EXEC, since whatever a command - * emits is always wrapped around MULTI/EXEC. */ - if (ctx->flags & REDISMODULE_CTX_MULTI_EMITTED) { - robj *propargv[1]; - propargv[0] = createStringObject("EXEC",4); - alsoPropagate(server.execCommand,c->db->id,propargv,1, - PROPAGATE_AOF|PROPAGATE_REPL); - decrRefCount(propargv[0]); - } -} - /* This Redis command binds the normal Redis command invocation with commands * exported by modules. */ void RedisModuleCommandDispatcher(client *c) { RedisModuleCommandProxy *cp = (void*)(unsigned long)c->cmd->getkeys_proc; RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.flags |= REDISMODULE_CTX_MODULE_COMMAND_CALL; ctx.module = cp->module; ctx.client = c; cp->func(&ctx,(void**)c->argv,c->argc); - moduleHandlePropagationAfterCommandCallback(&ctx); moduleFreeContext(&ctx); /* In some cases processMultibulkBuffer uses sdsMakeRoomFor to @@ -616,9 +714,9 @@ void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) { * flags into the command flags used by the Redis core. * * It returns the set of flags, or -1 if unknown flags are found. */ -int commandFlagsFromString(char *s) { +int64_t commandFlagsFromString(char *s) { int count, j; - int flags = 0; + int64_t flags = 0; sds *tokens = sdssplitlen(s,strlen(s)," ",1,&count); for (j = 0; j < count; j++) { char *t = tokens[j]; @@ -632,7 +730,9 @@ int commandFlagsFromString(char *s) { else if (!strcasecmp(t,"random")) flags |= CMD_RANDOM; else if (!strcasecmp(t,"allow-stale")) flags |= CMD_STALE; else if (!strcasecmp(t,"no-monitor")) flags |= CMD_SKIP_MONITOR; + else if (!strcasecmp(t,"no-slowlog")) flags |= CMD_SKIP_SLOWLOG; else if (!strcasecmp(t,"fast")) flags |= CMD_FAST; + else if (!strcasecmp(t,"no-auth")) flags |= CMD_NO_AUTH; else if (!strcasecmp(t,"getkeys-api")) flags |= CMD_MODULE_GETKEYS; else if (!strcasecmp(t,"no-cluster")) flags |= CMD_MODULE_NO_CLUSTER; else break; @@ -682,6 +782,8 @@ int commandFlagsFromString(char *s) { * this means. * * **"no-monitor"**: Don't propagate the command on monitor. Use this if * the command has sensible data among the arguments. + * * **"no-slowlog"**: Don't log this command in the slowlog. Use this if + * the command has sensible data among the arguments. * * **"fast"**: The command time complexity is not greater * than O(log(N)) where N is the size of the collection or * anything else representing the normal scalability @@ -694,9 +796,12 @@ int commandFlagsFromString(char *s) { * example, is unable to report the position of the * keys, programmatically creates key names, or any * other reason. + * * **"no-auth"**: This command can be run by an un-authenticated client. + * Normally this is used by a command that is used + * to authenticate a client. */ int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) { - int flags = strflags ? commandFlagsFromString((char*)strflags) : 0; + int64_t flags = strflags ? commandFlagsFromString((char*)strflags) : 0; if (flags == -1) return REDISMODULE_ERR; if ((flags & CMD_MODULE_NO_CLUSTER) && server.cluster_enabled) return REDISMODULE_ERR; @@ -734,6 +839,7 @@ int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc c cp->rediscmd->calls = 0; dictAdd(server.commands,sdsdup(cmdname),cp->rediscmd); dictAdd(server.orig_commands,sdsdup(cmdname),cp->rediscmd); + cp->rediscmd->id = ACLGetCommandID(cmdname); /* ID used for ACL. */ return REDISMODULE_OK; } @@ -754,6 +860,9 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api module->using = listCreate(); module->filters = listCreate(); module->in_call = 0; + module->in_hook = 0; + module->options = 0; + module->info_cb = 0; ctx->module = module; } @@ -771,6 +880,26 @@ long long RM_Milliseconds(void) { return mstime(); } +/* Set flags defining capabilities or behavior bit flags. + * + * REDISMODULE_OPTIONS_HANDLE_IO_ERRORS: + * Generally, modules don't need to bother with this, as the process will just + * terminate if a read error happens, however, setting this flag would allow + * repl-diskless-load to work if enabled. + * The module should use RedisModule_IsIOError after reads, before using the + * data that was read, and in case of error, propagate it upwards, and also be + * able to release the partially populated value and all it's allocations. */ +void RM_SetModuleOptions(RedisModuleCtx *ctx, int options) { + ctx->module->options = options; +} + +/* Signals that the key is modified from user's perspective (i.e. invalidate WATCH + * and client side caching). */ +int RM_SignalModifiedKey(RedisModuleCtx *ctx, RedisModuleString *keyname) { + signalModifiedKey(ctx->client,ctx->client->db,keyname); + return REDISMODULE_OK; +} + /* -------------------------------------------------------------------------- * Automatic memory management for modules * -------------------------------------------------------------------------- */ @@ -846,6 +975,7 @@ void autoMemoryCollect(RedisModuleCtx *ctx) { case REDISMODULE_AM_REPLY: RM_FreeCallReply(ptr); break; case REDISMODULE_AM_KEY: RM_CloseKey(ptr); break; case REDISMODULE_AM_DICT: RM_FreeDict(NULL,ptr); break; + case REDISMODULE_AM_INFO: RM_FreeServerInfo(NULL,ptr); break; } } ctx->flags |= REDISMODULE_CTX_AUTO_MEMORY; @@ -912,6 +1042,32 @@ RedisModuleString *RM_CreateStringFromLongLong(RedisModuleCtx *ctx, long long ll return RM_CreateString(ctx,buf,len); } +/* Like RedisModule_CreatString(), but creates a string starting from a double + * integer instead of taking a buffer and its length. + * + * The returned string must be released with RedisModule_FreeString() or by + * enabling automatic memory management. */ +RedisModuleString *RM_CreateStringFromDouble(RedisModuleCtx *ctx, double d) { + char buf[128]; + size_t len = d2string(buf,sizeof(buf),d); + return RM_CreateString(ctx,buf,len); +} + +/* Like RedisModule_CreatString(), but creates a string starting from a long + * double. + * + * The returned string must be released with RedisModule_FreeString() or by + * enabling automatic memory management. + * + * The passed context 'ctx' may be NULL if necessary, see the + * RedisModule_CreateString() documentation for more info. */ +RedisModuleString *RM_CreateStringFromLongDouble(RedisModuleCtx *ctx, long double ld, int humanfriendly) { + char buf[MAX_LONG_DOUBLE_CHARS]; + size_t len = ld2string(buf,sizeof(buf),ld, + (humanfriendly ? LD_STR_HUMAN : LD_STR_AUTO)); + return RM_CreateString(ctx,buf,len); +} + /* Like RedisModule_CreatString(), but creates a string starting from another * RedisModuleString. * @@ -1016,6 +1172,14 @@ int RM_StringToDouble(const RedisModuleString *str, double *d) { return (retval == C_OK) ? REDISMODULE_OK : REDISMODULE_ERR; } +/* Convert the string into a long double, storing it at `*ld`. + * Returns REDISMODULE_OK on success or REDISMODULE_ERR if the string is + * not a valid string representation of a double value. */ +int RM_StringToLongDouble(const RedisModuleString *str, long double *ld) { + int retval = string2ld(str->ptr,sdslen(str->ptr),ld); + return retval ? REDISMODULE_OK : REDISMODULE_ERR; +} + /* Compare two string objects, returning -1, 0 or 1 respectively if * a < b, a == b, a > b. Strings are compared byte by byte as two * binary blobs without any encoding care / collation attempt. */ @@ -1125,10 +1289,9 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return REDISMODULE_OK; - sds strmsg = sdsnewlen(prefix,1); - strmsg = sdscat(strmsg,msg); - strmsg = sdscatlen(strmsg,"\r\n",2); - addReplySds(c,strmsg); + addReplyProto(c,prefix,strlen(prefix)); + addReplyProto(c,msg,strlen(msg)); + addReplyProto(c,"\r\n",2); return REDISMODULE_OK; } @@ -1177,14 +1340,35 @@ int RM_ReplyWithArray(RedisModuleCtx *ctx, long len) { ctx->postponed_arrays = zrealloc(ctx->postponed_arrays,sizeof(void*)* (ctx->postponed_arrays_count+1)); ctx->postponed_arrays[ctx->postponed_arrays_count] = - addDeferredMultiBulkLength(c); + addReplyDeferredLen(c); ctx->postponed_arrays_count++; } else { - addReplyMultiBulkLen(c,len); + addReplyArrayLen(c,len); } return REDISMODULE_OK; } +/* Reply to the client with a null array, simply null in RESP3 + * null array in RESP2. + * + * The function always returns REDISMODULE_OK. */ +int RM_ReplyWithNullArray(RedisModuleCtx *ctx) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + addReplyNullArray(c); + return REDISMODULE_OK; +} + +/* Reply to the client with an empty array. + * + * The function always returns REDISMODULE_OK. */ +int RM_ReplyWithEmptyArray(RedisModuleCtx *ctx) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + addReply(c,shared.emptyarray); + return REDISMODULE_OK; +} + /* When RedisModule_ReplyWithArray() is used with the argument * REDISMODULE_POSTPONED_ARRAY_LEN, because we don't know beforehand the number * of items we are going to output as elements of the array, this function @@ -1223,7 +1407,7 @@ void RM_ReplySetArrayLength(RedisModuleCtx *ctx, long len) { return; } ctx->postponed_arrays_count--; - setDeferredMultiBulkLength(c, + setDeferredArrayLen(c, ctx->postponed_arrays[ctx->postponed_arrays_count], len); if (ctx->postponed_arrays_count == 0) { @@ -1263,14 +1447,34 @@ int RM_ReplyWithString(RedisModuleCtx *ctx, RedisModuleString *str) { return REDISMODULE_OK; } -/* Reply to the client with a NULL. In the RESP protocol a NULL is encoded - * as the string "$-1\r\n". +/* Reply with an empty string. + * + * The function always returns REDISMODULE_OK. */ +int RM_ReplyWithEmptyString(RedisModuleCtx *ctx) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + addReply(c,shared.emptybulk); + return REDISMODULE_OK; +} + +/* Reply with a binary safe string, which should not be escaped or filtered + * taking in input a C buffer pointer and length. + * + * The function always returns REDISMODULE_OK. */ +int RM_ReplyWithVerbatimString(RedisModuleCtx *ctx, const char *buf, size_t len) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + addReplyVerbatim(c, buf, len, "txt"); + return REDISMODULE_OK; +} + +/* Reply to the client with a NULL. * * The function always returns REDISMODULE_OK. */ int RM_ReplyWithNull(RedisModuleCtx *ctx) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return REDISMODULE_OK; - addReply(c,shared.nullbulk); + addReplyNull(c); return REDISMODULE_OK; } @@ -1301,6 +1505,21 @@ int RM_ReplyWithDouble(RedisModuleCtx *ctx, double d) { return REDISMODULE_OK; } +/* Send a string reply obtained converting the long double 'ld' into a bulk + * string. This function is basically equivalent to converting a long double + * into a string into a C buffer, and then calling the function + * RedisModule_ReplyWithStringBuffer() with the buffer and length. + * The double string uses human readable formatting (see + * `addReplyHumanLongDouble` in networking.c). + * + * The function always returns REDISMODULE_OK. */ +int RM_ReplyWithLongDouble(RedisModuleCtx *ctx, long double ld) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + addReplyHumanLongDouble(c, ld); + return REDISMODULE_OK; +} + /* -------------------------------------------------------------------------- * Commands replication API * -------------------------------------------------------------------------- */ @@ -1315,9 +1534,16 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { /* If we already emitted MULTI return ASAP. */ if (ctx->flags & REDISMODULE_CTX_MULTI_EMITTED) return; /* If this is a thread safe context, we do not want to wrap commands - * executed into MUTLI/EXEC, they are executed as single commands + * executed into MULTI/EXEC, they are executed as single commands * from an external client in essence. */ if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) return; + /* If this is a callback context, and not a module command execution + * context, we have to setup the op array for the "also propagate" API + * so that RM_Replicate() will work. */ + if (!(ctx->flags & REDISMODULE_CTX_MODULE_COMMAND_CALL)) { + ctx->saved_oparray = server.also_propagate; + redisOpArrayInit(&server.also_propagate); + } execCommandPropagateMulti(ctx->client); ctx->flags |= REDISMODULE_CTX_MULTI_EMITTED; } @@ -1339,6 +1565,24 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { * * Please refer to RedisModule_Call() for more information. * + * Using the special "A" and "R" modifiers, the caller can exclude either + * the AOF or the replicas from the propagation of the specified command. + * Otherwise, by default, the command will be propagated in both channels. + * + * ## Note about calling this function from a thread safe context: + * + * Normally when you call this function from the callback implementing a + * module command, or any other callback provided by the Redis Module API, + * Redis will accumulate all the calls to this function in the context of + * the callback, and will propagate all the commands wrapped in a MULTI/EXEC + * transaction. However when calling this function from a threaded safe context + * that can live an undefined amount of time, and can be locked/unlocked in + * at will, the behavior is different: MULTI/EXEC wrapper is not emitted + * and the command specified is inserted in the AOF and replication stream + * immediately. + * + * ## Return value + * * The command returns REDISMODULE_ERR if the format specifiers are invalid * or the command name does not belong to a known command. */ int RM_Replicate(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) { @@ -1356,10 +1600,23 @@ int RM_Replicate(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) va_end(ap); if (argv == NULL) return REDISMODULE_ERR; - /* Replicate! */ - moduleReplicateMultiIfNeeded(ctx); - alsoPropagate(cmd,ctx->client->db->id,argv,argc, - PROPAGATE_AOF|PROPAGATE_REPL); + /* Select the propagation target. Usually is AOF + replicas, however + * the caller can exclude one or the other using the "A" or "R" + * modifiers. */ + int target = 0; + if (!(flags & REDISMODULE_ARGV_NO_AOF)) target |= PROPAGATE_AOF; + if (!(flags & REDISMODULE_ARGV_NO_REPLICAS)) target |= PROPAGATE_REPL; + + /* Replicate! When we are in a threaded context, we want to just insert + * the replicated command ASAP, since it is not clear when the context + * will stop being used, so accumulating stuff does not make much sense, + * nor we could easily use the alsoPropagate() API from threads. */ + if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) { + propagate(cmd,ctx->client->db->id,argv,argc,target); + } else { + moduleReplicateMultiIfNeeded(ctx); + alsoPropagate(cmd,ctx->client->db->id,argv,argc,target); + } /* Release the argv. */ for (j = 0; j < argc; j++) decrRefCount(argv[j]); @@ -1401,12 +1658,133 @@ int RM_ReplicateVerbatim(RedisModuleCtx *ctx) { * are guaranteed to get IDs greater than any past ID previously seen. * * Valid IDs are from 1 to 2^64-1. If 0 is returned it means there is no way - * to fetch the ID in the context the function was currently called. */ + * to fetch the ID in the context the function was currently called. + * + * After obtaining the ID, it is possible to check if the command execution + * is actually happening in the context of AOF loading, using this macro: + * + * if (RedisModule_IsAOFClient(RedisModule_GetClientId(ctx)) { + * // Handle it differently. + * } + */ unsigned long long RM_GetClientId(RedisModuleCtx *ctx) { if (ctx->client == NULL) return 0; return ctx->client->id; } +/* This is an helper for RM_GetClientInfoById() and other functions: given + * a client, it populates the client info structure with the appropriate + * fields depending on the version provided. If the version is not valid + * then REDISMODULE_ERR is returned. Otherwise the function returns + * REDISMODULE_OK and the structure pointed by 'ci' gets populated. */ + +int modulePopulateClientInfoStructure(void *ci, client *client, int structver) { + if (structver != 1) return REDISMODULE_ERR; + + RedisModuleClientInfoV1 *ci1 = ci; + memset(ci1,0,sizeof(*ci1)); + ci1->version = structver; + if (client->flags & CLIENT_MULTI) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_MULTI; + if (client->flags & CLIENT_PUBSUB) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_PUBSUB; + if (client->flags & CLIENT_UNIX_SOCKET) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_UNIXSOCKET; + if (client->flags & CLIENT_TRACKING) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_TRACKING; + if (client->flags & CLIENT_BLOCKED) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_BLOCKED; + + int port; + connPeerToString(client->conn,ci1->addr,sizeof(ci1->addr),&port); + ci1->port = port; + ci1->db = client->db->id; + ci1->id = client->id; + return REDISMODULE_OK; +} + +/* This is an helper for moduleFireServerEvent() and other functions: + * It populates the replication info structure with the appropriate + * fields depending on the version provided. If the version is not valid + * then REDISMODULE_ERR is returned. Otherwise the function returns + * REDISMODULE_OK and the structure pointed by 'ri' gets populated. */ +int modulePopulateReplicationInfoStructure(void *ri, int structver) { + if (structver != 1) return REDISMODULE_ERR; + + RedisModuleReplicationInfoV1 *ri1 = ri; + memset(ri1,0,sizeof(*ri1)); + ri1->version = structver; + ri1->master = server.masterhost==NULL; + ri1->masterhost = server.masterhost? server.masterhost: ""; + ri1->masterport = server.masterport; + ri1->replid1 = server.replid; + ri1->replid2 = server.replid2; + ri1->repl1_offset = server.master_repl_offset; + ri1->repl2_offset = server.second_replid_offset; + return REDISMODULE_OK; +} + +/* Return information about the client with the specified ID (that was + * previously obtained via the RedisModule_GetClientId() API). If the + * client exists, REDISMODULE_OK is returned, otherwise REDISMODULE_ERR + * is returned. + * + * When the client exist and the `ci` pointer is not NULL, but points to + * a structure of type RedisModuleClientInfo, previously initialized with + * the correct REDISMODULE_CLIENTINFO_INITIALIZER, the structure is populated + * with the following fields: + * + * uint64_t flags; // REDISMODULE_CLIENTINFO_FLAG_* + * uint64_t id; // Client ID + * char addr[46]; // IPv4 or IPv6 address. + * uint16_t port; // TCP port. + * uint16_t db; // Selected DB. + * + * Note: the client ID is useless in the context of this call, since we + * already know, however the same structure could be used in other + * contexts where we don't know the client ID, yet the same structure + * is returned. + * + * With flags having the following meaning: + * + * REDISMODULE_CLIENTINFO_FLAG_SSL Client using SSL connection. + * REDISMODULE_CLIENTINFO_FLAG_PUBSUB Client in Pub/Sub mode. + * REDISMODULE_CLIENTINFO_FLAG_BLOCKED Client blocked in command. + * REDISMODULE_CLIENTINFO_FLAG_TRACKING Client with keys tracking on. + * REDISMODULE_CLIENTINFO_FLAG_UNIXSOCKET Client using unix domain socket. + * REDISMODULE_CLIENTINFO_FLAG_MULTI Client in MULTI state. + * + * However passing NULL is a way to just check if the client exists in case + * we are not interested in any additional information. + * + * This is the correct usage when we want the client info structure + * returned: + * + * RedisModuleClientInfo ci = REDISMODULE_CLIENTINFO_INITIALIZER; + * int retval = RedisModule_GetClientInfoById(&ci,client_id); + * if (retval == REDISMODULE_OK) { + * printf("Address: %s\n", ci.addr); + * } + */ +int RM_GetClientInfoById(void *ci, uint64_t id) { + client *client = lookupClientByID(id); + if (client == NULL) return REDISMODULE_ERR; + if (ci == NULL) return REDISMODULE_OK; + + /* Fill the info structure if passed. */ + uint64_t structver = ((uint64_t*)ci)[0]; + return modulePopulateClientInfoStructure(ci,client,structver); +} + +/* Publish a message to subscribers (see PUBLISH command). */ +int RM_PublishMessage(RedisModuleCtx *ctx, RedisModuleString *channel, RedisModuleString *message) { + UNUSED(ctx); + int receivers = pubsubPublishMessage(channel, message); + if (server.cluster_enabled) + clusterPropagatePublish(channel, message); + return receivers; +} + /* Return the currently selected DB. */ int RM_GetSelectedDb(RedisModuleCtx *ctx) { return ctx->client->db->id; @@ -1417,7 +1795,12 @@ int RM_GetSelectedDb(RedisModuleCtx *ctx) { * current request context (whether the client is a Lua script or in a MULTI), * and about the Redis instance in general, i.e replication and persistence. * - * The available flags are: + * It is possible to call this function even with a NULL context, however + * in this case the following flags will not be reported: + * + * * LUA, MULTI, REPLICATED, DIRTY (see below for more info). + * + * Available flags and their meaning: * * * REDISMODULE_CTX_FLAGS_LUA: The command is running in a Lua script * @@ -1448,19 +1831,44 @@ int RM_GetSelectedDb(RedisModuleCtx *ctx) { * * * REDISMODULE_CTX_FLAGS_OOM_WARNING: Less than 25% of memory remains before * reaching the maxmemory level. + * + * * REDISMODULE_CTX_FLAGS_LOADING: Server is loading RDB/AOF + * + * * REDISMODULE_CTX_FLAGS_REPLICA_IS_STALE: No active link with the master. + * + * * REDISMODULE_CTX_FLAGS_REPLICA_IS_CONNECTING: The replica is trying to + * connect with the master. + * + * * REDISMODULE_CTX_FLAGS_REPLICA_IS_TRANSFERRING: Master -> Replica RDB + * transfer is in progress. + * + * * REDISMODULE_CTX_FLAGS_REPLICA_IS_ONLINE: The replica has an active link + * with its master. This is the + * contrary of STALE state. + * + * * REDISMODULE_CTX_FLAGS_ACTIVE_CHILD: There is currently some background + * process active (RDB, AUX or module). */ int RM_GetContextFlags(RedisModuleCtx *ctx) { int flags = 0; /* Client specific flags */ - if (ctx->client) { - if (ctx->client->flags & CLIENT_LUA) - flags |= REDISMODULE_CTX_FLAGS_LUA; - if (ctx->client->flags & CLIENT_MULTI) - flags |= REDISMODULE_CTX_FLAGS_MULTI; - /* Module command recieved from MASTER, is replicated. */ - if (ctx->client->flags & CLIENT_MASTER) - flags |= REDISMODULE_CTX_FLAGS_REPLICATED; + if (ctx) { + if (ctx->client) { + if (ctx->client->flags & CLIENT_LUA) + flags |= REDISMODULE_CTX_FLAGS_LUA; + if (ctx->client->flags & CLIENT_MULTI) + flags |= REDISMODULE_CTX_FLAGS_MULTI; + /* Module command recieved from MASTER, is replicated. */ + if (ctx->client->flags & CLIENT_MASTER) + flags |= REDISMODULE_CTX_FLAGS_REPLICATED; + } + + /* For DIRTY flags, we need the blocked client if used */ + client *c = ctx->blocked_client ? ctx->blocked_client->client : ctx->client; + if (c && (c->flags & (CLIENT_DIRTY_CAS|CLIENT_DIRTY_EXEC))) { + flags |= REDISMODULE_CTX_FLAGS_MULTI_DIRTY; + } } if (server.cluster_enabled) @@ -1490,6 +1898,20 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) { flags |= REDISMODULE_CTX_FLAGS_SLAVE; if (server.repl_slave_ro) flags |= REDISMODULE_CTX_FLAGS_READONLY; + + /* Replica state flags. */ + if (server.repl_state == REPL_STATE_CONNECT || + server.repl_state == REPL_STATE_CONNECTING) + { + flags |= REDISMODULE_CTX_FLAGS_REPLICA_IS_CONNECTING; + } else if (server.repl_state == REPL_STATE_TRANSFER) { + flags |= REDISMODULE_CTX_FLAGS_REPLICA_IS_TRANSFERRING; + } else if (server.repl_state == REPL_STATE_CONNECTED) { + flags |= REDISMODULE_CTX_FLAGS_REPLICA_IS_ONLINE; + } + + if (server.repl_state != REPL_STATE_CONNECTED) + flags |= REDISMODULE_CTX_FLAGS_REPLICA_IS_STALE; } /* OOM flag. */ @@ -1498,9 +1920,36 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) { if (retval == C_ERR) flags |= REDISMODULE_CTX_FLAGS_OOM; if (level > 0.75) flags |= REDISMODULE_CTX_FLAGS_OOM_WARNING; + /* Presence of children processes. */ + if (hasActiveChildProcess()) flags |= REDISMODULE_CTX_FLAGS_ACTIVE_CHILD; + return flags; } +/* Returns true if some client sent the CLIENT PAUSE command to the server or + * if Redis Cluster is doing a manual failover, and paused tue clients. + * This is needed when we have a master with replicas, and want to write, + * without adding further data to the replication channel, that the replicas + * replication offset, match the one of the master. When this happens, it is + * safe to failover the master without data loss. + * + * However modules may generate traffic by calling RedisModule_Call() with + * the "!" flag, or by calling RedisModule_Replicate(), in a context outside + * commands execution, for instance in timeout callbacks, threads safe + * contexts, and so forth. When modules will generate too much traffic, it + * will be hard for the master and replicas offset to match, because there + * is more data to send in the replication channel. + * + * So modules may want to try to avoid very heavy background work that has + * the effect of creating data to the replication channel, when this function + * returns true. This is mostly useful for modules that have background + * garbage collection tasks, or that do writes and replicate such writes + * periodically in timer callbacks or other periodic callbacks. + */ +int RM_AvoidReplicaTraffic() { + return clientsArePaused(); +} + /* Change the currently selected DB. Returns an error if the id * is out of range. * @@ -1516,6 +1965,18 @@ int RM_SelectDb(RedisModuleCtx *ctx, int newid) { return (retval == C_OK) ? REDISMODULE_OK : REDISMODULE_ERR; } +/* Initialize a RedisModuleKey struct */ +static void moduleInitKey(RedisModuleKey *kp, RedisModuleCtx *ctx, robj *keyname, robj *value, int mode){ + kp->ctx = ctx; + kp->db = ctx->client->db; + kp->key = keyname; + incrRefCount(keyname); + kp->value = value; + kp->iter = NULL; + kp->mode = mode; + zsetKeyReset(kp); +} + /* Return an handle representing a Redis key, so that it is possible * to call other APIs with the key handle as argument to perform * operations on the key. @@ -1533,11 +1994,12 @@ int RM_SelectDb(RedisModuleCtx *ctx, int newid) { void *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) { RedisModuleKey *kp; robj *value; + int flags = mode & REDISMODULE_OPEN_KEY_NOTOUCH? LOOKUP_NOTOUCH: 0; if (mode & REDISMODULE_WRITE) { - value = lookupKeyWrite(ctx->client->db,keyname); + value = lookupKeyWriteWithFlags(ctx->client->db,keyname, flags); } else { - value = lookupKeyRead(ctx->client->db,keyname); + value = lookupKeyReadWithFlags(ctx->client->db,keyname, flags); if (value == NULL) { return NULL; } @@ -1545,25 +2007,25 @@ void *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) { /* Setup the key handle. */ kp = zmalloc(sizeof(*kp)); - kp->ctx = ctx; - kp->db = ctx->client->db; - kp->key = keyname; - incrRefCount(keyname); - kp->value = value; - kp->iter = NULL; - kp->mode = mode; - zsetKeyReset(kp); + moduleInitKey(kp, ctx, keyname, value, mode); autoMemoryAdd(ctx,REDISMODULE_AM_KEY,kp); return (void*)kp; } -/* Close a key handle. */ -void RM_CloseKey(RedisModuleKey *key) { - if (key == NULL) return; - if (key->mode & REDISMODULE_WRITE) signalModifiedKey(key->db,key->key); +/* Destroy a RedisModuleKey struct (freeing is the responsibility of the caller). */ +static void moduleCloseKey(RedisModuleKey *key) { + int signal = SHOULD_SIGNAL_MODIFIED_KEYS(key->ctx); + if ((key->mode & REDISMODULE_WRITE) && signal) + signalModifiedKey(key->ctx->client,key->db,key->key); /* TODO: if (key->iter) RM_KeyIteratorStop(kp); */ RM_ZsetRangeStop(key); decrRefCount(key->key); +} + +/* Close a key handle. */ +void RM_CloseKey(RedisModuleKey *key) { + if (key == NULL) return; + moduleCloseKey(key); autoMemoryFreed(key->ctx,REDISMODULE_AM_KEY,key); zfree(key); } @@ -1581,6 +2043,7 @@ int RM_KeyType(RedisModuleKey *key) { case OBJ_ZSET: return REDISMODULE_KEYTYPE_ZSET; case OBJ_HASH: return REDISMODULE_KEYTYPE_HASH; case OBJ_MODULE: return REDISMODULE_KEYTYPE_MODULE; + case OBJ_STREAM: return REDISMODULE_KEYTYPE_STREAM; default: return 0; } } @@ -1598,6 +2061,7 @@ size_t RM_ValueLength(RedisModuleKey *key) { case OBJ_SET: return setTypeSize(key->value); case OBJ_ZSET: return zsetLength(key->value); case OBJ_HASH: return hashTypeLength(key->value); + case OBJ_STREAM: return streamLength(key->value); default: return 0; } } @@ -1615,7 +2079,7 @@ int RM_DeleteKey(RedisModuleKey *key) { return REDISMODULE_OK; } -/* If the key is open for writing, unlink it (that is delete it in a +/* If the key is open for writing, unlink it (that is delete it in a * non-blocking way, not reclaiming memory immediately) and setup the key to * accept new writes as an empty key (that will be created on demand). * On success REDISMODULE_OK is returned. If the key is not open for @@ -1660,6 +2124,28 @@ int RM_SetExpire(RedisModuleKey *key, mstime_t expire) { return REDISMODULE_OK; } +/* Performs similar operation to FLUSHALL, and optionally start a new AOF file (if enabled) + * If restart_aof is true, you must make sure the command that triggered this call is not + * propagated to the AOF file. + * When async is set to true, db contents will be freed by a background thread. */ +void RM_ResetDataset(int restart_aof, int async) { + if (restart_aof && server.aof_state != AOF_OFF) stopAppendOnly(); + flushAllDataAndResetRDB(async? EMPTYDB_ASYNC: EMPTYDB_NO_FLAGS); + if (server.aof_enabled && restart_aof) restartAOFAfterSYNC(); +} + +/* Returns the number of keys in the current db. */ +unsigned long long RM_DbSize(RedisModuleCtx *ctx) { + return dictSize(ctx->client->db->dict); +} + +/* Returns a name of a random key, or NULL if current db is empty. */ +RedisModuleString *RM_RandomKey(RedisModuleCtx *ctx) { + robj *key = dbRandomKey(ctx->client->db); + autoMemoryAdd(ctx,REDISMODULE_AM_STRING,key); + return key; +} + /* -------------------------------------------------------------------------- * Key API for String type * -------------------------------------------------------------------------- */ @@ -1671,7 +2157,7 @@ int RM_SetExpire(RedisModuleKey *key, mstime_t expire) { int RM_StringSet(RedisModuleKey *key, RedisModuleString *str) { if (!(key->mode & REDISMODULE_WRITE) || key->iter) return REDISMODULE_ERR; RM_DeleteKey(key); - setKey(key->db,key->key,str); + genericSetKey(key->ctx->client,key->db,key->key,str,0,0); key->value = str; return REDISMODULE_OK; } @@ -1751,7 +2237,7 @@ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) { if (key->value == NULL) { /* Empty key: create it with the new size. */ robj *o = createObject(OBJ_STRING,sdsnewlen(NULL, newlen)); - setKey(key->db,key->key,o); + genericSetKey(key->ctx->client,key->db,key->key,o,0,0); key->value = o; decrRefCount(o); } else { @@ -2388,7 +2874,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) { * * REDISMODULE_HASH_EXISTS: instead of setting the value of the field * expecting a RedisModuleString pointer to pointer, the function just - * reports if the field esists or not and expects an integer pointer + * reports if the field exists or not and expects an integer pointer * as the second element of each pair. * * Example of REDISMODULE_HASH_CFIELD: @@ -2677,12 +3163,11 @@ RedisModuleString *RM_CreateStringFromCallReply(RedisModuleCallReply *reply) { * to special modifiers in "fmt". For now only one exists: * * "!" -> REDISMODULE_ARGV_REPLICATE + * "A" -> REDISMODULE_ARGV_NO_AOF + * "R" -> REDISMODULE_ARGV_NO_REPLICAS * * On error (format specifier error) NULL is returned and nothing is * allocated. On success the argument vector is returned. */ - -#define REDISMODULE_ARGV_REPLICATE (1<<0) - robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int *argcp, int *flags, va_list ap) { int argc = 0, argv_size, j; robj **argv = NULL; @@ -2711,7 +3196,7 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int size_t len = va_arg(ap,size_t); argv[argc++] = createStringObject(buf,len); } else if (*p == 'l') { - long ll = va_arg(ap,long long); + long long ll = va_arg(ap,long long); argv[argc++] = createObject(OBJ_STRING,sdsfromlonglong(ll)); } else if (*p == 'v') { /* A vector of strings */ @@ -2731,6 +3216,10 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int } } else if (*p == '!') { if (flags) (*flags) |= REDISMODULE_ARGV_REPLICATE; + } else if (*p == 'A') { + if (flags) (*flags) |= REDISMODULE_ARGV_NO_AOF; + } else if (*p == 'R') { + if (flags) (*flags) |= REDISMODULE_ARGV_NO_REPLICAS; } else { goto fmterr; } @@ -2750,8 +3239,16 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int * On success a RedisModuleCallReply object is returned, otherwise * NULL is returned and errno is set to the following values: * - * EINVAL: command non existing, wrong arity, wrong format specifier. - * EPERM: operation in Cluster instance with key in non local slot. */ + * EBADF: wrong format specifier. + * EINVAL: wrong command arity. + * ENOENT: command does not exist. + * EPERM: operation in Cluster instance with key in non local slot. + * EROFS: operation in Cluster instance when a write command is sent + * in a readonly state. + * ENETDOWN: operation in Cluster instance when cluster is down. + * + * This API is documented here: https://redis.io/topics/modules-intro + */ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) { struct redisCommand *cmd; client *c = NULL; @@ -2763,7 +3260,8 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch /* Create the client and dispatch the command. */ va_start(ap, fmt); - c = createClient(-1); + c = createClient(NULL); + c->user = NULL; /* Root user. */ argv = moduleCreateArgvFromUserFormat(cmdname,fmt,&argc,&flags,ap); replicate = flags & REDISMODULE_ARGV_REPLICATE; va_end(ap); @@ -2777,7 +3275,10 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch /* We handle the above format error only when the client is setup so that * we can free it normally. */ - if (argv == NULL) goto cleanup; + if (argv == NULL) { + errno = EBADF; + goto cleanup; + } /* Call command filters */ moduleCallCommandFilters(c); @@ -2787,7 +3288,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch */ cmd = lookupCommand(c->argv[0]->ptr); if (!cmd) { - errno = EINVAL; + errno = ENOENT; goto cleanup; } c->cmd = c->lastcmd = cmd; @@ -2802,13 +3303,20 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch * trying to access non-local keys, with the exception of commands * received from our master. */ if (server.cluster_enabled && !(ctx->client->flags & CLIENT_MASTER)) { + int error_code; /* Duplicate relevant flags in the module client. */ c->flags &= ~(CLIENT_READONLY|CLIENT_ASKING); c->flags |= ctx->client->flags & (CLIENT_READONLY|CLIENT_ASKING); - if (getNodeByQuery(c,c->cmd,c->argv,c->argc,NULL,NULL) != + if (getNodeByQuery(c,c->cmd,c->argv,c->argc,NULL,&error_code) != server.cluster->myself) { - errno = EPERM; + if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) { + errno = EROFS; + } else if (error_code == CLUSTER_REDIR_DOWN_STATE) { + errno = ENETDOWN; + } else { + errno = EPERM; + } goto cleanup; } } @@ -2819,16 +3327,16 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch if (replicate) moduleReplicateMultiIfNeeded(ctx); /* Run the command */ - int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS; + int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS | CMD_CALL_NOWRAP; if (replicate) { - call_flags |= CMD_CALL_PROPAGATE_AOF; - call_flags |= CMD_CALL_PROPAGATE_REPL; + if (!(flags & REDISMODULE_ARGV_NO_AOF)) + call_flags |= CMD_CALL_PROPAGATE_AOF; + if (!(flags & REDISMODULE_ARGV_NO_REPLICAS)) + call_flags |= CMD_CALL_PROPAGATE_REPL; } call(c,call_flags); - /* Convert the result of the Redis command into a suitable Lua type. - * The first thing we need is to create a single string from the client - * output buffers. */ + /* Convert the result of the Redis command into a module reply. */ sds proto = sdsnewlen(c->buf,c->bufpos); c->bufpos = 0; while(listLength(c->reply)) { @@ -3032,6 +3540,8 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) { * // Optional fields * .digest = myType_DigestCallBack, * .mem_usage = myType_MemUsageCallBack, + * .aux_load = myType_AuxRDBLoadCallBack, + * .aux_save = myType_AuxRDBSaveCallBack, * } * * * **rdb_load**: A callback function pointer that loads data from RDB files. @@ -3039,6 +3549,10 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) { * * **aof_rewrite**: A callback function pointer that rewrites data as commands. * * **digest**: A callback function pointer that is used for `DEBUG DIGEST`. * * **free**: A callback function pointer that can free a type value. + * * **aux_save**: A callback function pointer that saves out of keyspace data to RDB files. + * 'when' argument is either REDISMODULE_AUX_BEFORE_RDB or REDISMODULE_AUX_AFTER_RDB. + * * **aux_load**: A callback function pointer that loads out of keyspace data from RDB files. + * Similar to aux_save, returns REDISMODULE_OK on success, and ERR otherwise. * * The **digest* and **mem_usage** methods should currently be omitted since * they are not yet implemented inside the Redis modules core. @@ -3111,7 +3625,7 @@ int RM_ModuleTypeSetValue(RedisModuleKey *key, moduleType *mt, void *value) { if (!(key->mode & REDISMODULE_WRITE) || key->iter) return REDISMODULE_ERR; RM_DeleteKey(key); robj *o = createModuleObject(mt,value); - setKey(key->db,key->key,o); + genericSetKey(key->ctx->client,key->db,key->key,o,0,0); decrRefCount(o); key->value = o; return REDISMODULE_OK; @@ -3148,17 +3662,50 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) { * RDB loading and saving functions * -------------------------------------------------------------------------- */ -/* Called when there is a load error in the context of a module. This cannot - * be recovered like for the built-in types. */ +/* Called when there is a load error in the context of a module. On some + * modules this cannot be recovered, but if the module declared capability + * to handle errors, we'll raise a flag rather than exiting. */ void moduleRDBLoadError(RedisModuleIO *io) { - serverLog(LL_WARNING, + if (io->type->module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) { + io->error = 1; + return; + } + serverPanic( "Error loading data from RDB (short read or EOF). " "Read performed by module '%s' about type '%s' " - "after reading '%llu' bytes of a value.", + "after reading '%llu' bytes of a value " + "for key named: '%s'.", io->type->module->name, io->type->name, - (unsigned long long)io->bytes); - exit(1); + (unsigned long long)io->bytes, + io->key? (char*)io->key->ptr: "(null)"); +} + +/* Returns 0 if there's at least one registered data type that did not declare + * REDISMODULE_OPTIONS_HANDLE_IO_ERRORS, in which case diskless loading should + * be avoided since it could cause data loss. */ +int moduleAllDatatypesHandleErrors() { + dictIterator *di = dictGetIterator(modules); + dictEntry *de; + + while ((de = dictNext(di)) != NULL) { + struct RedisModule *module = dictGetVal(de); + if (listLength(module->types) && + !(module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS)) + { + dictReleaseIterator(di); + return 0; + } + } + dictReleaseIterator(di); + return 1; +} + +/* Returns true if any previous IO API failed. + * for Load* APIs the REDISMODULE_OPTIONS_HANDLE_IO_ERRORS flag must be set with + * RediModule_SetModuleOptions first. */ +int RM_IsIOError(RedisModuleIO *io) { + return io->error; } /* Save an unsigned 64 bit value into the RDB file. This function should only @@ -3184,6 +3731,7 @@ void RM_SaveUnsigned(RedisModuleIO *io, uint64_t value) { * be called in the context of the rdb_load method of modules implementing * new data types. */ uint64_t RM_LoadUnsigned(RedisModuleIO *io) { + if (io->error) return 0; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_UINT) goto loaderr; @@ -3195,7 +3743,7 @@ uint64_t RM_LoadUnsigned(RedisModuleIO *io) { loaderr: moduleRDBLoadError(io); - return 0; /* Never reached. */ + return 0; } /* Like RedisModule_SaveUnsigned() but for signed 64 bit values. */ @@ -3254,6 +3802,7 @@ void RM_SaveStringBuffer(RedisModuleIO *io, const char *str, size_t len) { /* Implements RM_LoadString() and RM_LoadStringBuffer() */ void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) { + if (io->error) return NULL; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_STRING) goto loaderr; @@ -3265,7 +3814,7 @@ void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) { loaderr: moduleRDBLoadError(io); - return NULL; /* Never reached. */ + return NULL; } /* In the context of the rdb_load method of a module data type, loads a string @@ -3286,7 +3835,7 @@ RedisModuleString *RM_LoadString(RedisModuleIO *io) { * RedisModule_Realloc() or RedisModule_Free(). * * The size of the string is stored at '*lenptr' if not NULL. - * The returned string is not automatically NULL termianted, it is loaded + * The returned string is not automatically NULL terminated, it is loaded * exactly as it was stored inisde the RDB file. */ char *RM_LoadStringBuffer(RedisModuleIO *io, size_t *lenptr) { return moduleLoadString(io,1,lenptr); @@ -3314,6 +3863,7 @@ void RM_SaveDouble(RedisModuleIO *io, double value) { /* In the context of the rdb_save method of a module data type, loads back the * double value saved by RedisModule_SaveDouble(). */ double RM_LoadDouble(RedisModuleIO *io) { + if (io->error) return 0; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_DOUBLE) goto loaderr; @@ -3325,7 +3875,7 @@ double RM_LoadDouble(RedisModuleIO *io) { loaderr: moduleRDBLoadError(io); - return 0; /* Never reached. */ + return 0; } /* In the context of the rdb_save method of a module data type, saves a float @@ -3350,6 +3900,7 @@ void RM_SaveFloat(RedisModuleIO *io, float value) { /* In the context of the rdb_save method of a module data type, loads back the * float value saved by RedisModule_SaveFloat(). */ float RM_LoadFloat(RedisModuleIO *io) { + if (io->error) return 0; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_FLOAT) goto loaderr; @@ -3361,7 +3912,32 @@ float RM_LoadFloat(RedisModuleIO *io) { loaderr: moduleRDBLoadError(io); - return 0; /* Never reached. */ + return 0; +} + +/* In the context of the rdb_save method of a module data type, saves a long double + * value to the RDB file. The double can be a valid number, a NaN or infinity. + * It is possible to load back the value with RedisModule_LoadLongDouble(). */ +void RM_SaveLongDouble(RedisModuleIO *io, long double value) { + if (io->error) return; + char buf[MAX_LONG_DOUBLE_CHARS]; + /* Long double has different number of bits in different platforms, so we + * save it as a string type. */ + size_t len = ld2string(buf,sizeof(buf),value,LD_STR_HEX); + RM_SaveStringBuffer(io,buf,len); +} + +/* In the context of the rdb_save method of a module data type, loads back the + * long double value saved by RedisModule_SaveLongDouble(). */ +long double RM_LoadLongDouble(RedisModuleIO *io) { + if (io->error) return 0; + long double value; + size_t len; + char* str = RM_LoadStringBuffer(io,&len); + if (!str) return 0; + string2ld(str,len,&value); + RM_Free(str); + return value; } /* Iterate over modules, and trigger rdb aux saving for the ones modules types @@ -3454,21 +4030,84 @@ void RM_DigestEndSequence(RedisModuleDigest *md) { memset(md->o,0,sizeof(md->o)); } -/* -------------------------------------------------------------------------- - * AOF API for modules data types - * -------------------------------------------------------------------------- */ +/* Decode a serialized representation of a module data type 'mt' from string + * 'str' and return a newly allocated value, or NULL if decoding failed. + * + * This call basically reuses the 'rdb_load' callback which module data types + * implement in order to allow a module to arbitrarily serialize/de-serialize + * keys, similar to how the Redis 'DUMP' and 'RESTORE' commands are implemented. + * + * Modules should generally use the REDISMODULE_OPTIONS_HANDLE_IO_ERRORS flag and + * make sure the de-serialization code properly checks and handles IO errors + * (freeing allocated buffers and returning a NULL). + * + * If this is NOT done, Redis will handle corrupted (or just truncated) serialized + * data by producing an error message and terminating the process. + */ -/* Emits a command into the AOF during the AOF rewriting process. This function - * is only called in the context of the aof_rewrite method of data types exported - * by a module. The command works exactly like RedisModule_Call() in the way - * the parameters are passed, but it does not return anything as the error - * handling is performed by Redis itself. */ -void RM_EmitAOF(RedisModuleIO *io, const char *cmdname, const char *fmt, ...) { - if (io->error) return; - struct redisCommand *cmd; - robj **argv = NULL; - int argc = 0, flags = 0, j; - va_list ap; +void *RM_LoadDataTypeFromString(const RedisModuleString *str, const moduleType *mt) { + rio payload; + RedisModuleIO io; + void *ret; + + rioInitWithBuffer(&payload, str->ptr); + moduleInitIOContext(io,(moduleType *)mt,&payload,NULL); + + /* All RM_Save*() calls always write a version 2 compatible format, so we + * need to make sure we read the same. + */ + io.ver = 2; + ret = mt->rdb_load(&io,0); + if (io.ctx) { + moduleFreeContext(io.ctx); + zfree(io.ctx); + } + return ret; +} + +/* Encode a module data type 'mt' value 'data' into serialized form, and return it + * as a newly allocated RedisModuleString. + * + * This call basically reuses the 'rdb_save' callback which module data types + * implement in order to allow a module to arbitrarily serialize/de-serialize + * keys, similar to how the Redis 'DUMP' and 'RESTORE' commands are implemented. + */ + +RedisModuleString *RM_SaveDataTypeToString(RedisModuleCtx *ctx, void *data, const moduleType *mt) { + rio payload; + RedisModuleIO io; + + rioInitWithBuffer(&payload,sdsempty()); + moduleInitIOContext(io,(moduleType *)mt,&payload,NULL); + mt->rdb_save(&io,data); + if (io.ctx) { + moduleFreeContext(io.ctx); + zfree(io.ctx); + } + if (io.error) { + return NULL; + } else { + robj *str = createObject(OBJ_STRING,payload.io.buffer.ptr); + if (ctx != NULL) autoMemoryAdd(ctx,REDISMODULE_AM_STRING,str); + return str; + } +} + +/* -------------------------------------------------------------------------- + * AOF API for modules data types + * -------------------------------------------------------------------------- */ + +/* Emits a command into the AOF during the AOF rewriting process. This function + * is only called in the context of the aof_rewrite method of data types exported + * by a module. The command works exactly like RedisModule_Call() in the way + * the parameters are passed, but it does not return anything as the error + * handling is performed by Redis itself. */ +void RM_EmitAOF(RedisModuleIO *io, const char *cmdname, const char *fmt, ...) { + if (io->error) return; + struct redisCommand *cmd; + robj **argv = NULL; + int argc = 0, flags = 0, j; + va_list ap; cmd = lookupCommandByCString((char*)cmdname); if (!cmd) { @@ -3531,6 +4170,11 @@ const RedisModuleString *RM_GetKeyNameFromIO(RedisModuleIO *io) { return io->key; } +/* Returns a RedisModuleString with the name of the key from RedisModuleKey */ +const RedisModuleString *RM_GetKeyNameFromModuleKey(RedisModuleKey *key) { + return key ? key->key : NULL; +} + /* -------------------------------------------------------------------------- * Logging * -------------------------------------------------------------------------- */ @@ -3596,6 +4240,23 @@ void RM_LogIOError(RedisModuleIO *io, const char *levelstr, const char *fmt, ... va_end(ap); } +/* Redis-like assert function. + * + * A failed assertion will shut down the server and produce logging information + * that looks identical to information generated by Redis itself. + */ +void RM__Assert(const char *estr, const char *file, int line) { + _serverAssert(estr, file, line); +} + +/* Allows adding event to the latency monitor to be observed by the LATENCY + * command. The call is skipped if the latency is smaller than the configured + * latency-monitor-threshold. */ +void RM_LatencyAddSample(const char *event, mstime_t latency) { + if (latency >= server.latency_monitor_threshold) + latencyAddSample(event, latency); +} + /* -------------------------------------------------------------------------- * Blocking clients from modules * -------------------------------------------------------------------------- */ @@ -3626,7 +4287,10 @@ void moduleBlockedClientPipeReadable(aeEventLoop *el, int fd, void *privdata, in void unblockClientFromModule(client *c) { RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; - /* Call the disconnection callback if any. */ + /* Call the disconnection callback if any. Note that + * bc->disconnect_callback is set to NULL if the client gets disconnected + * by the module itself or because of a timeout, so the callback will NOT + * get called if this is not an actual disconnection event. */ if (bc->disconnect_callback) { RedisModuleCtx ctx = REDISMODULE_CTX_INIT; ctx.blocked_privdata = bc->privdata; @@ -3636,6 +4300,24 @@ void unblockClientFromModule(client *c) { moduleFreeContext(&ctx); } + /* If we made it here and client is still blocked it means that the command + * timed-out, client was killed or disconnected and disconnect_callback was + * not implemented (or it was, but RM_UnblockClient was not called from + * within it, as it should). + * We must call moduleUnblockClient in order to free privdata and + * RedisModuleBlockedClient. + * + * Note that we only do that for clients that are blocked on keys, for which + * the contract is that the module should not call RM_UnblockClient under + * normal circumstances. + * Clients implementing threads and working with private data should be + * aware that calling RM_UnblockClient for every blocked client is their + * responsibility, and if they fail to do so memory may leak. Ideally they + * should implement the disconnect and timeout callbacks and call + * RM_UnblockClient, but any other way is also acceptable. */ + if (bc->blocked_on_keys && !bc->unblocked) + moduleUnblockClient(c); + bc->client = NULL; /* Reset the client for a new query since, for blocking commands implemented * into modules, we do not it immediately after the command returns (and @@ -3644,45 +4326,52 @@ void unblockClientFromModule(client *c) { resetClient(c); } -/* Block a client in the context of a blocking command, returning an handle - * which will be used, later, in order to unblock the client with a call to - * RedisModule_UnblockClient(). The arguments specify callback functions - * and a timeout after which the client is unblocked. - * - * The callbacks are called in the following contexts: +/* Block a client in the context of a module: this function implements both + * RM_BlockClient() and RM_BlockClientOnKeys() depending on the fact the + * keys are passed or not. * - * reply_callback: called after a successful RedisModule_UnblockClient() - * call in order to reply to the client and unblock it. + * When not blocking for keys, the keys, numkeys, and privdata parameters are + * not needed. The privdata in that case must be NULL, since later is + * RM_UnblockClient() that will provide some private data that the reply + * callback will receive. * - * reply_timeout: called when the timeout is reached in order to send an - * error to the client. + * Instead when blocking for keys, normally RM_UnblockClient() will not be + * called (because the client will unblock when the key is modified), so + * 'privdata' should be provided in that case, so that once the client is + * unlocked and the reply callback is called, it will receive its associated + * private data. * - * free_privdata: called in order to free the private data that is passed - * by RedisModule_UnblockClient() call. + * Even when blocking on keys, RM_UnblockClient() can be called however, but + * in that case the privdata argument is disregarded, because we pass the + * reply callback the privdata that is set here while blocking. */ -RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms) { +RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata) { client *c = ctx->client; int islua = c->flags & CLIENT_LUA; int ismulti = c->flags & CLIENT_MULTI; c->bpop.module_blocked_handle = zmalloc(sizeof(RedisModuleBlockedClient)); RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; + ctx->module->blocked_clients++; /* We need to handle the invalid operation of calling modules blocking * commands from Lua or MULTI. We actually create an already aborted * (client set to NULL) blocked client handle, and actually reply with * an error. */ + mstime_t timeout = timeout_ms ? (mstime()+timeout_ms) : 0; bc->client = (islua || ismulti) ? NULL : c; bc->module = ctx->module; bc->reply_callback = reply_callback; bc->timeout_callback = timeout_callback; bc->disconnect_callback = NULL; /* Set by RM_SetDisconnectCallback() */ bc->free_privdata = free_privdata; - bc->privdata = NULL; - bc->reply_client = createClient(-1); + bc->privdata = privdata; + bc->reply_client = createClient(NULL); bc->reply_client->flags |= CLIENT_MODULE; bc->dbid = c->db->id; - c->bpop.timeout = timeout_ms ? (mstime()+timeout_ms) : 0; + bc->blocked_on_keys = keys != NULL; + bc->unblocked = 0; + c->bpop.timeout = timeout; if (islua || ismulti) { c->bpop.module_blocked_handle = NULL; @@ -3690,11 +4379,165 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc "Blocking module command called from Lua script" : "Blocking module command called from transaction"); } else { - blockClient(c,BLOCKED_MODULE); + if (keys) { + blockForKeys(c,BLOCKED_MODULE,keys,numkeys,timeout,NULL,NULL); + } else { + blockClient(c,BLOCKED_MODULE); + } } return bc; } +/* This function is called from module.c in order to check if a module + * blocked for BLOCKED_MODULE and subtype 'on keys' (bc->blocked_on_keys true) + * can really be unblocked, since the module was able to serve the client. + * If the callback returns REDISMODULE_OK, then the client can be unblocked, + * otherwise the client remains blocked and we'll retry again when one of + * the keys it blocked for becomes "ready" again. + * This function returns 1 if client was served (and should be unblocked) */ +int moduleTryServeClientBlockedOnKey(client *c, robj *key) { + int served = 0; + RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; + + /* Protect against re-processing: don't serve clients that are already + * in the unblocking list for any reason (including RM_UnblockClient() + * explicit call). See #6798. */ + if (bc->unblocked) return 0; + + RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.flags |= REDISMODULE_CTX_BLOCKED_REPLY; + ctx.blocked_ready_key = key; + ctx.blocked_privdata = bc->privdata; + ctx.module = bc->module; + ctx.client = bc->client; + ctx.blocked_client = bc; + if (bc->reply_callback(&ctx,(void**)c->argv,c->argc) == REDISMODULE_OK) + served = 1; + moduleFreeContext(&ctx); + return served; +} + +/* Block a client in the context of a blocking command, returning an handle + * which will be used, later, in order to unblock the client with a call to + * RedisModule_UnblockClient(). The arguments specify callback functions + * and a timeout after which the client is unblocked. + * + * The callbacks are called in the following contexts: + * + * reply_callback: called after a successful RedisModule_UnblockClient() + * call in order to reply to the client and unblock it. + * + * reply_timeout: called when the timeout is reached in order to send an + * error to the client. + * + * free_privdata: called in order to free the private data that is passed + * by RedisModule_UnblockClient() call. + * + * Note: RedisModule_UnblockClient should be called for every blocked client, + * even if client was killed, timed-out or disconnected. Failing to do so + * will result in memory leaks. + */ +RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms) { + return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, NULL,0,NULL); +} + +/* This call is similar to RedisModule_BlockClient(), however in this case we + * don't just block the client, but also ask Redis to unblock it automatically + * once certain keys become "ready", that is, contain more data. + * + * Basically this is similar to what a typical Redis command usually does, + * like BLPOP or ZPOPMAX: the client blocks if it cannot be served ASAP, + * and later when the key receives new data (a list push for instance), the + * client is unblocked and served. + * + * However in the case of this module API, when the client is unblocked? + * + * 1. If you block ok a key of a type that has blocking operations associated, + * like a list, a sorted set, a stream, and so forth, the client may be + * unblocked once the relevant key is targeted by an operation that normally + * unblocks the native blocking operations for that type. So if we block + * on a list key, an RPUSH command may unblock our client and so forth. + * 2. If you are implementing your native data type, or if you want to add new + * unblocking conditions in addition to "1", you can call the modules API + * RedisModule_SignalKeyAsReady(). + * + * Anyway we can't be sure if the client should be unblocked just because the + * key is signaled as ready: for instance a successive operation may change the + * key, or a client in queue before this one can be served, modifying the key + * as well and making it empty again. So when a client is blocked with + * RedisModule_BlockClientOnKeys() the reply callback is not called after + * RM_UnblockCLient() is called, but every time a key is signaled as ready: + * if the reply callback can serve the client, it returns REDISMODULE_OK + * and the client is unblocked, otherwise it will return REDISMODULE_ERR + * and we'll try again later. + * + * The reply callback can access the key that was signaled as ready by + * calling the API RedisModule_GetBlockedClientReadyKey(), that returns + * just the string name of the key as a RedisModuleString object. + * + * Thanks to this system we can setup complex blocking scenarios, like + * unblocking a client only if a list contains at least 5 items or other + * more fancy logics. + * + * Note that another difference with RedisModule_BlockClient(), is that here + * we pass the private data directly when blocking the client: it will + * be accessible later in the reply callback. Normally when blocking with + * RedisModule_BlockClient() the private data to reply to the client is + * passed when calling RedisModule_UnblockClient() but here the unblocking + * is performed by Redis itself, so we need to have some private data before + * hand. The private data is used to store any information about the specific + * unblocking operation that you are implementing. Such information will be + * freed using the free_privdata callback provided by the user. + * + * However the reply callback will be able to access the argument vector of + * the command, so the private data is often not needed. + * + * Note: Under normal circumstances RedisModule_UnblockClient should not be + * called for clients that are blocked on keys (Either the key will + * become ready or a timeout will occur). If for some reason you do want + * to call RedisModule_UnblockClient it is possible: Client will be + * handled as if it were timed-out (You must implement the timeout + * callback in that case). + */ +RedisModuleBlockedClient *RM_BlockClientOnKeys(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata) { + return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, keys,numkeys,privdata); +} + +/* This function is used in order to potentially unblock a client blocked + * on keys with RedisModule_BlockClientOnKeys(). When this function is called, + * all the clients blocked for this key will get their reply callback called, + * and if the callback returns REDISMODULE_OK the client will be unblocked. */ +void RM_SignalKeyAsReady(RedisModuleCtx *ctx, RedisModuleString *key) { + signalKeyAsReady(ctx->client->db, key); +} + +/* Implements RM_UnblockClient() and moduleUnblockClient(). */ +int moduleUnblockClientByHandle(RedisModuleBlockedClient *bc, void *privdata) { + pthread_mutex_lock(&moduleUnblockedClientsMutex); + if (!bc->blocked_on_keys) bc->privdata = privdata; + bc->unblocked = 1; + listAddNodeTail(moduleUnblockedClients,bc); + if (write(server.module_blocked_pipe[1],"A",1) != 1) { + /* Ignore the error, this is best-effort. */ + } + pthread_mutex_unlock(&moduleUnblockedClientsMutex); + return REDISMODULE_OK; +} + +/* This API is used by the Redis core to unblock a client that was blocked + * by a module. */ +void moduleUnblockClient(client *c) { + RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; + moduleUnblockClientByHandle(bc,NULL); +} + +/* Return true if the client 'c' was blocked by a module using + * RM_BlockClientOnKeys(). */ +int moduleClientIsBlockedOnKeys(client *c) { + RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; + return bc->blocked_on_keys; +} + /* Unblock a client blocked by `RedisModule_BlockedClient`. This will trigger * the reply callbacks to be called in order to reply to the client. * The 'privdata' argument will be accessible by the reply callback, so @@ -3705,15 +4548,25 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc * needs to be passed to the client, included but not limited some slow * to compute reply or some reply obtained via networking. * - * Note: this function can be called from threads spawned by the module. */ + * Note 1: this function can be called from threads spawned by the module. + * + * Note 2: when we unblock a client that is blocked for keys using + * the API RedisModule_BlockClientOnKeys(), the privdata argument here is + * not used, and the reply callback is called with the privdata pointer that + * was passed when blocking the client. + * + * Unblocking a client that was blocked for keys using this API will still + * require the client to get some reply, so the function will use the + * "timeout" handler in order to do so. */ int RM_UnblockClient(RedisModuleBlockedClient *bc, void *privdata) { - pthread_mutex_lock(&moduleUnblockedClientsMutex); - bc->privdata = privdata; - listAddNodeTail(moduleUnblockedClients,bc); - if (write(server.module_blocked_pipe[1],"A",1) != 1) { - /* Ignore the error, this is best-effort. */ + if (bc->blocked_on_keys) { + /* In theory the user should always pass the timeout handler as an + * argument, but better to be safe than sorry. */ + if (bc->timeout_callback == NULL) return REDISMODULE_ERR; + if (bc->unblocked) return REDISMODULE_OK; + if (bc->client) moduleBlockedClientTimedOut(bc->client); } - pthread_mutex_unlock(&moduleUnblockedClientsMutex); + moduleUnblockClientByHandle(bc,privdata); return REDISMODULE_OK; } @@ -3773,16 +4626,19 @@ void moduleHandleBlockedClients(void) { * touch the shared list. */ /* Call the reply callback if the client is valid and we have - * any callback. */ - if (c && bc->reply_callback) { + * any callback. However the callback is not called if the client + * was blocked on keys (RM_BlockClientOnKeys()), because we already + * called such callback in moduleTryServeClientBlockedOnKey() when + * the key was signaled as ready. */ + if (c && !bc->blocked_on_keys && bc->reply_callback) { RedisModuleCtx ctx = REDISMODULE_CTX_INIT; ctx.flags |= REDISMODULE_CTX_BLOCKED_REPLY; ctx.blocked_privdata = bc->privdata; + ctx.blocked_ready_key = NULL; ctx.module = bc->module; ctx.client = bc->client; ctx.blocked_client = bc; bc->reply_callback(&ctx,(void**)c->argv,c->argc); - moduleHandlePropagationAfterCommandCallback(&ctx); moduleFreeContext(&ctx); } @@ -3825,6 +4681,7 @@ void moduleHandleBlockedClients(void) { /* Free 'bc' only after unblocking the client, since it is * referenced in the client blocking context, and must be valid * when calling unblockClient(). */ + bc->module->blocked_clients--; zfree(bc); /* Lock again before to iterate the loop. */ @@ -3869,6 +4726,12 @@ void *RM_GetBlockedClientPrivateData(RedisModuleCtx *ctx) { return ctx->blocked_privdata; } +/* Get the key that is ready when the reply callback is called in the context + * of a client blocked by RedisModule_BlockClientOnKeys(). */ +RedisModuleString *RM_GetBlockedClientReadyKey(RedisModuleCtx *ctx) { + return ctx->blocked_ready_key; +} + /* Get the blocked client associated with a given context. * This is useful in the reply and timeout callbacks of blocked clients, * before sometimes the module has the blocked client handle references @@ -3897,9 +4760,9 @@ int RM_BlockedClientDisconnected(RedisModuleCtx *ctx) { * * To call non-reply APIs, the thread safe context must be prepared with: * - * RedisModule_ThreadSafeCallStart(ctx); + * RedisModule_ThreadSafeContextLock(ctx); * ... make your call here ... - * RedisModule_ThreadSafeCallStop(ctx); + * RedisModule_ThreadSafeContextUnlock(ctx); * * This is not needed when using `RedisModule_Reply*` functions, assuming * that a blocked client was used when the context was created, otherwise @@ -3920,10 +4783,10 @@ RedisModuleCtx *RM_GetThreadSafeContext(RedisModuleBlockedClient *bc) { * access it safely from another thread, so we create a fake client here * in order to keep things like the currently selected database and similar * things. */ - ctx->client = createClient(-1); + ctx->client = createClient(NULL); if (bc) { selectDb(ctx->client,bc->dbid); - ctx->client->id = bc->client->id; + if (bc->client) ctx->client->id = bc->client->id; } return ctx; } @@ -3982,7 +4845,8 @@ void moduleReleaseGIL(void) { * - REDISMODULE_NOTIFY_EXPIRED: Expiration events * - REDISMODULE_NOTIFY_EVICTED: Eviction events * - REDISMODULE_NOTIFY_STREAM: Stream events - * - REDISMODULE_NOTIFY_ALL: All events + * - REDISMODULE_NOTIFY_KEYMISS: Key-miss events + * - REDISMODULE_NOTIFY_ALL: All events (Excluding REDISMODULE_NOTIFY_KEYMISS) * * We do not distinguish between key events and keyspace events, and it is up * to the module to filter the actions taken based on the key. @@ -4021,6 +4885,20 @@ int RM_SubscribeToKeyspaceEvents(RedisModuleCtx *ctx, int types, RedisModuleNoti return REDISMODULE_OK; } +/* Get the configured bitmap of notify-keyspace-events (Could be used + * for additional filtering in RedisModuleNotificationFunc) */ +int RM_GetNotifyKeyspaceEvents() { + return server.notify_keyspace_events; +} + +/* Expose notifyKeyspaceEvent to modules */ +int RM_NotifyKeyspaceEvent(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key) { + if (!ctx || !ctx->client) + return REDISMODULE_ERR; + notifyKeyspaceEvent(type, (char *)event, key, ctx->client->db->id); + return REDISMODULE_OK; +} + /* Dispatcher for keyspace notifications to module subscriber functions. * This gets called only if at least one module requested to be notified on * keyspace notifications */ @@ -4267,10 +5145,13 @@ int RM_GetClusterNodeInfo(RedisModuleCtx *ctx, const char *id, char *ip, char *m UNUSED(ctx); clusterNode *node = clusterLookupNode(id); - if (node->flags & (CLUSTER_NODE_NOADDR|CLUSTER_NODE_HANDSHAKE)) + if (node == NULL || + node->flags & (CLUSTER_NODE_NOADDR|CLUSTER_NODE_HANDSHAKE)) + { return REDISMODULE_ERR; + } - if (ip) memcpy(ip,node->name,REDISMODULE_NODE_ID_LEN); + if (ip) strncpy(ip,node->ip,NET_IP_STR_LEN); if (master_id) { /* If the information is not available, the function will set the @@ -4404,7 +5285,7 @@ RedisModuleTimerID RM_CreateTimer(RedisModuleCtx *ctx, mstime_t period, RedisMod timer->module = ctx->module; timer->callback = callback; timer->data = data; - timer->dbid = ctx->client->db->id; + timer->dbid = ctx->client ? ctx->client->db->id : 0; uint64_t expiretime = ustime()+period*1000; uint64_t key; @@ -4476,6 +5357,200 @@ int RM_GetTimerInfo(RedisModuleCtx *ctx, RedisModuleTimerID id, uint64_t *remain return REDISMODULE_OK; } +/* -------------------------------------------------------------------------- + * Modules ACL API + * + * Implements a hook into the authentication and authorization within Redis. + * --------------------------------------------------------------------------*/ + +/* This function is called when a client's user has changed and invokes the + * client's user changed callback if it was set. This callback should + * cleanup any state the module was tracking about this client. + * + * A client's user can be changed through the AUTH command, module + * authentication, and when a client is freed. */ +void moduleNotifyUserChanged(client *c) { + if (c->auth_callback) { + c->auth_callback(c->id, c->auth_callback_privdata); + + /* The callback will fire exactly once, even if the user remains + * the same. It is expected to completely clean up the state + * so all references are cleared here. */ + c->auth_callback = NULL; + c->auth_callback_privdata = NULL; + c->auth_module = NULL; + } +} + +void revokeClientAuthentication(client *c) { + /* Freeing the client would result in moduleNotifyUserChanged() to be + * called later, however since we use revokeClientAuthentication() also + * in moduleFreeAuthenticatedClients() to implement module unloading, we + * do this action ASAP: this way if the module is unloaded, when the client + * is eventually freed we don't rely on the module to still exist. */ + moduleNotifyUserChanged(c); + + c->user = DefaultUser; + c->authenticated = 0; + freeClientAsync(c); +} + +/* Cleanup all clients that have been authenticated with this module. This + * is called from onUnload() to give the module a chance to cleanup any + * resources associated with clients it has authenticated. */ +static void moduleFreeAuthenticatedClients(RedisModule *module) { + listIter li; + listNode *ln; + listRewind(server.clients,&li); + while ((ln = listNext(&li)) != NULL) { + client *c = listNodeValue(ln); + if (!c->auth_module) continue; + + RedisModule *auth_module = (RedisModule *) c->auth_module; + if (auth_module == module) { + revokeClientAuthentication(c); + } + } +} + +/* Creates a Redis ACL user that the module can use to authenticate a client. + * After obtaining the user, the module should set what such user can do + * using the RM_SetUserACL() function. Once configured, the user + * can be used in order to authenticate a connection, with the specified + * ACL rules, using the RedisModule_AuthClientWithUser() function. + * + * Note that: + * + * * Users created here are not listed by the ACL command. + * * Users created here are not checked for duplicated name, so it's up to + * the module calling this function to take care of not creating users + * with the same name. + * * The created user can be used to authenticate multiple Redis connections. + * + * The caller can later free the user using the function + * RM_FreeModuleUser(). When this function is called, if there are + * still clients authenticated with this user, they are disconnected. + * The function to free the user should only be used when the caller really + * wants to invalidate the user to define a new one with different + * capabilities. */ +RedisModuleUser *RM_CreateModuleUser(const char *name) { + RedisModuleUser *new_user = zmalloc(sizeof(RedisModuleUser)); + new_user->user = ACLCreateUnlinkedUser(); + + /* Free the previous temporarily assigned name to assign the new one */ + sdsfree(new_user->user->name); + new_user->user->name = sdsnew(name); + return new_user; +} + +/* Frees a given user and disconnects all of the clients that have been + * authenticated with it. See RM_CreateModuleUser for detailed usage.*/ +int RM_FreeModuleUser(RedisModuleUser *user) { + ACLFreeUserAndKillClients(user->user); + zfree(user); + return REDISMODULE_OK; +} + +/* Sets the permissions of a user created through the redis module + * interface. The syntax is the same as ACL SETUSER, so refer to the + * documentation in acl.c for more information. See RM_CreateModuleUser + * for detailed usage. + * + * Returns REDISMODULE_OK on success and REDISMODULE_ERR on failure + * and will set an errno describing why the operation failed. */ +int RM_SetModuleUserACL(RedisModuleUser *user, const char* acl) { + return ACLSetUser(user->user, acl, -1); +} + +/* Authenticate the client associated with the context with + * the provided user. Returns REDISMODULE_OK on success and + * REDISMODULE_ERR on error. + * + * This authentication can be tracked with the optional callback and private + * data fields. The callback will be called whenever the user of the client + * changes. This callback should be used to cleanup any state that is being + * kept in the module related to the client authentication. It will only be + * called once, even when the user hasn't changed, in order to allow for a + * new callback to be specified. If this authentication does not need to be + * tracked, pass in NULL for the callback and privdata. + * + * If client_id is not NULL, it will be filled with the id of the client + * that was authenticated. This can be used with the + * RM_DeauthenticateAndCloseClient() API in order to deauthenticate a + * previously authenticated client if the authentication is no longer valid. + * + * For expensive authentication operations, it is recommended to block the + * client and do the authentication in the background and then attach the user + * to the client in a threadsafe context. */ +static int authenticateClientWithUser(RedisModuleCtx *ctx, user *user, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id) { + if (user->flags & USER_FLAG_DISABLED) { + return REDISMODULE_ERR; + } + + moduleNotifyUserChanged(ctx->client); + + ctx->client->user = user; + ctx->client->authenticated = 1; + + if (callback) { + ctx->client->auth_callback = callback; + ctx->client->auth_callback_privdata = privdata; + ctx->client->auth_module = ctx->module; + } + + if (client_id) { + *client_id = ctx->client->id; + } + + return REDISMODULE_OK; +} + + +/* Authenticate the current context's user with the provided redis acl user. + * Returns REDISMODULE_ERR if the user is disabled. + * + * See authenticateClientWithUser for information about callback, client_id, + * and general usage for authentication. */ +int RM_AuthenticateClientWithUser(RedisModuleCtx *ctx, RedisModuleUser *module_user, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id) { + return authenticateClientWithUser(ctx, module_user->user, callback, privdata, client_id); +} + +/* Authenticate the current context's user with the provided redis acl user. + * Returns REDISMODULE_ERR if the user is disabled or the user does not exist. + * + * See authenticateClientWithUser for information about callback, client_id, + * and general usage for authentication. */ +int RM_AuthenticateClientWithACLUser(RedisModuleCtx *ctx, const char *name, size_t len, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id) { + user *acl_user = ACLGetUserByName(name, len); + + if (!acl_user) { + return REDISMODULE_ERR; + } + return authenticateClientWithUser(ctx, acl_user, callback, privdata, client_id); +} + +/* Deauthenticate and close the client. The client resources will not be + * be immediately freed, but will be cleaned up in a background job. This is + * the recommended way to deauthenicate a client since most clients can't + * handle users becomming deauthenticated. Returns REDISMODULE_ERR when the + * client doesn't exist and REDISMODULE_OK when the operation was successful. + * + * The client ID is returned from the RM_AuthenticateClientWithUser and + * RM_AuthenticateClientWithACLUser APIs, but can be obtained through + * the CLIENT api or through server events. + * + * This function is not thread safe, and must be executed within the context + * of a command or thread safe context. */ +int RM_DeauthenticateAndCloseClient(RedisModuleCtx *ctx, uint64_t client_id) { + UNUSED(ctx); + client *c = lookupClientByID(client_id); + if (c == NULL) return REDISMODULE_ERR; + + /* Revoke also marks client to be closed ASAP */ + revokeClientAuthentication(c); + return REDISMODULE_OK; +} + /* -------------------------------------------------------------------------- * Modules Dictionary API * @@ -4727,86 +5802,394 @@ int RM_DictCompare(RedisModuleDictIter *di, const char *op, RedisModuleString *k return res ? REDISMODULE_OK : REDISMODULE_ERR; } + + + /* -------------------------------------------------------------------------- - * Modules utility APIs + * Modules Info fields * -------------------------------------------------------------------------- */ -/* Return random bytes using SHA1 in counter mode with a /dev/urandom - * initialized seed. This function is fast so can be used to generate - * many bytes without any effect on the operating system entropy pool. - * Currently this function is not thread safe. */ -void RM_GetRandomBytes(unsigned char *dst, size_t len) { - getRandomBytes(dst,len); +int RM_InfoEndDictField(RedisModuleInfoCtx *ctx); + +/* Used to start a new section, before adding any fields. the section name will + * be prefixed by "_" and must only include A-Z,a-z,0-9. + * NULL or empty string indicates the default section (only ) is used. + * When return value is REDISMODULE_ERR, the section should and will be skipped. */ +int RM_InfoAddSection(RedisModuleInfoCtx *ctx, char *name) { + sds full_name = sdsdup(ctx->module->name); + if (name != NULL && strlen(name) > 0) + full_name = sdscatfmt(full_name, "_%s", name); + + /* Implicitly end dicts, instead of returning an error which is likely un checked. */ + if (ctx->in_dict_field) + RM_InfoEndDictField(ctx); + + /* proceed only if: + * 1) no section was requested (emit all) + * 2) the module name was requested (emit all) + * 3) this specific section was requested. */ + if (ctx->requested_section) { + if (strcasecmp(ctx->requested_section, full_name) && + strcasecmp(ctx->requested_section, ctx->module->name)) { + sdsfree(full_name); + ctx->in_section = 0; + return REDISMODULE_ERR; + } + } + if (ctx->sections++) ctx->info = sdscat(ctx->info,"\r\n"); + ctx->info = sdscatfmt(ctx->info, "# %S\r\n", full_name); + ctx->in_section = 1; + sdsfree(full_name); + return REDISMODULE_OK; } -/* Like RedisModule_GetRandomBytes() but instead of setting the string to - * random bytes the string is set to random characters in the in the - * hex charset [0-9a-f]. */ -void RM_GetRandomHexChars(char *dst, size_t len) { - getRandomHexChars(dst,len); +/* Starts a dict field, similar to the ones in INFO KEYSPACE. Use normal + * RedisModule_InfoAddField* functions to add the items to this field, and + * terminate with RedisModule_InfoEndDictField. */ +int RM_InfoBeginDictField(RedisModuleInfoCtx *ctx, char *name) { + if (!ctx->in_section) + return REDISMODULE_ERR; + /* Implicitly end dicts, instead of returning an error which is likely un checked. */ + if (ctx->in_dict_field) + RM_InfoEndDictField(ctx); + ctx->info = sdscatfmt(ctx->info, + "%s_%s:", + ctx->module->name, + name); + ctx->in_dict_field = 1; + return REDISMODULE_OK; } -/* -------------------------------------------------------------------------- - * Modules API exporting / importing - * -------------------------------------------------------------------------- */ +/* Ends a dict field, see RedisModule_InfoBeginDictField */ +int RM_InfoEndDictField(RedisModuleInfoCtx *ctx) { + if (!ctx->in_dict_field) + return REDISMODULE_ERR; + /* trim the last ',' if found. */ + if (ctx->info[sdslen(ctx->info)-1]==',') + sdsIncrLen(ctx->info, -1); + ctx->info = sdscat(ctx->info, "\r\n"); + ctx->in_dict_field = 0; + return REDISMODULE_OK; +} -/* This function is called by a module in order to export some API with a - * given name. Other modules will be able to use this API by calling the - * symmetrical function RM_GetSharedAPI() and casting the return value to - * the right function pointer. - * - * The function will return REDISMODULE_OK if the name is not already taken, - * otherwise REDISMODULE_ERR will be returned and no operation will be - * performed. - * - * IMPORTANT: the apiname argument should be a string literal with static - * lifetime. The API relies on the fact that it will always be valid in - * the future. */ -int RM_ExportSharedAPI(RedisModuleCtx *ctx, const char *apiname, void *func) { - RedisModuleSharedAPI *sapi = zmalloc(sizeof(*sapi)); - sapi->module = ctx->module; - sapi->func = func; - if (dictAdd(server.sharedapi, (char*)apiname, sapi) != DICT_OK) { - zfree(sapi); +/* Used by RedisModuleInfoFunc to add info fields. + * Each field will be automatically prefixed by "_". + * Field names or values must not include \r\n of ":" */ +int RM_InfoAddFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value) { + if (!ctx->in_section) return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatfmt(ctx->info, + "%s=%S,", + field, + (sds)value->ptr); + return REDISMODULE_OK; } + ctx->info = sdscatfmt(ctx->info, + "%s_%s:%S\r\n", + ctx->module->name, + field, + (sds)value->ptr); return REDISMODULE_OK; } -/* Request an exported API pointer. The return value is just a void pointer - * that the caller of this function will be required to cast to the right - * function pointer, so this is a private contract between modules. - * - * If the requested API is not available then NULL is returned. Because - * modules can be loaded at different times with different order, this - * function calls should be put inside some module generic API registering - * step, that is called every time a module attempts to execute a - * command that requires external APIs: if some API cannot be resolved, the - * command should return an error. - * - * Here is an exmaple: - * - * int ... myCommandImplementation() { - * if (getExternalAPIs() == 0) { - * reply with an error here if we cannot have the APIs - * } - * // Use the API: - * myFunctionPointer(foo); - * } - * - * And the function registerAPI() is: - * - * int getExternalAPIs(void) { - * static int api_loaded = 0; - * if (api_loaded != 0) return 1; // APIs already resolved. - * - * myFunctionPointer = RedisModule_GetOtherModuleAPI("..."); - * if (myFunctionPointer == NULL) return 0; - * - * return 1; - * } - */ -void *RM_GetSharedAPI(RedisModuleCtx *ctx, const char *apiname) { +int RM_InfoAddFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatfmt(ctx->info, + "%s=%s,", + field, + value); + return REDISMODULE_OK; + } + ctx->info = sdscatfmt(ctx->info, + "%s_%s:%s\r\n", + ctx->module->name, + field, + value); + return REDISMODULE_OK; +} + +int RM_InfoAddFieldDouble(RedisModuleInfoCtx *ctx, char *field, double value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatprintf(ctx->info, + "%s=%.17g,", + field, + value); + return REDISMODULE_OK; + } + ctx->info = sdscatprintf(ctx->info, + "%s_%s:%.17g\r\n", + ctx->module->name, + field, + value); + return REDISMODULE_OK; +} + +int RM_InfoAddFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatfmt(ctx->info, + "%s=%I,", + field, + value); + return REDISMODULE_OK; + } + ctx->info = sdscatfmt(ctx->info, + "%s_%s:%I\r\n", + ctx->module->name, + field, + value); + return REDISMODULE_OK; +} + +int RM_InfoAddFieldULongLong(RedisModuleInfoCtx *ctx, char *field, unsigned long long value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatfmt(ctx->info, + "%s=%U,", + field, + value); + return REDISMODULE_OK; + } + ctx->info = sdscatfmt(ctx->info, + "%s_%s:%U\r\n", + ctx->module->name, + field, + value); + return REDISMODULE_OK; +} + +int RM_RegisterInfoFunc(RedisModuleCtx *ctx, RedisModuleInfoFunc cb) { + ctx->module->info_cb = cb; + return REDISMODULE_OK; +} + +sds modulesCollectInfo(sds info, const char *section, int for_crash_report, int sections) { + dictIterator *di = dictGetIterator(modules); + dictEntry *de; + + while ((de = dictNext(di)) != NULL) { + struct RedisModule *module = dictGetVal(de); + if (!module->info_cb) + continue; + RedisModuleInfoCtx info_ctx = {module, section, info, sections, 0, 0}; + module->info_cb(&info_ctx, for_crash_report); + /* Implicitly end dicts (no way to handle errors, and we must add the newline). */ + if (info_ctx.in_dict_field) + RM_InfoEndDictField(&info_ctx); + info = info_ctx.info; + sections = info_ctx.sections; + } + dictReleaseIterator(di); + return info; +} + +/* Get information about the server similar to the one that returns from the + * INFO command. This function takes an optional 'section' argument that may + * be NULL. The return value holds the output and can be used with + * RedisModule_ServerInfoGetField and alike to get the individual fields. + * When done, it needs to be freed with RedisModule_FreeServerInfo or with the + * automatic memory management mechanism if enabled. */ +RedisModuleServerInfoData *RM_GetServerInfo(RedisModuleCtx *ctx, const char *section) { + struct RedisModuleServerInfoData *d = zmalloc(sizeof(*d)); + d->rax = raxNew(); + if (ctx != NULL) autoMemoryAdd(ctx,REDISMODULE_AM_INFO,d); + sds info = genRedisInfoString(section); + int totlines, i; + sds *lines = sdssplitlen(info, sdslen(info), "\r\n", 2, &totlines); + for(i=0; irax,key,keylen,val,NULL)) + sdsfree(val); + } + sdsfree(info); + sdsfreesplitres(lines,totlines); + return d; +} + +/* Free data created with RM_GetServerInfo(). You need to pass the + * context pointer 'ctx' only if the dictionary was created using the + * context instead of passing NULL. */ +void RM_FreeServerInfo(RedisModuleCtx *ctx, RedisModuleServerInfoData *data) { + if (ctx != NULL) autoMemoryFreed(ctx,REDISMODULE_AM_INFO,data); + raxIterator ri; + raxStart(&ri,data->rax); + while(1) { + raxSeek(&ri,"^",NULL,0); + if (!raxNext(&ri)) break; + raxRemove(data->rax,(unsigned char*)ri.key,ri.key_len,NULL); + sdsfree(ri.data); + } + raxStop(&ri); + raxFree(data->rax); + zfree(data); +} + +/* Get the value of a field from data collected with RM_GetServerInfo(). You + * need to pass the context pointer 'ctx' only if you want to use auto memory + * mechanism to release the returned string. Return value will be NULL if the + * field was not found. */ +RedisModuleString *RM_ServerInfoGetField(RedisModuleCtx *ctx, RedisModuleServerInfoData *data, const char* field) { + sds val = raxFind(data->rax, (unsigned char *)field, strlen(field)); + if (val == raxNotFound) return NULL; + RedisModuleString *o = createStringObject(val,sdslen(val)); + if (ctx != NULL) autoMemoryAdd(ctx,REDISMODULE_AM_STRING,o); + return o; +} + +/* Similar to RM_ServerInfoGetField, but returns a char* which should not be freed but the caller. */ +const char *RM_ServerInfoGetFieldC(RedisModuleServerInfoData *data, const char* field) { + sds val = raxFind(data->rax, (unsigned char *)field, strlen(field)); + if (val == raxNotFound) return NULL; + return val; +} + +/* Get the value of a field from data collected with RM_GetServerInfo(). If the + * field is not found, or is not numerical or out of range, return value will be + * 0, and the optional out_err argument will be set to REDISMODULE_ERR. */ +long long RM_ServerInfoGetFieldSigned(RedisModuleServerInfoData *data, const char* field, int *out_err) { + long long ll; + sds val = raxFind(data->rax, (unsigned char *)field, strlen(field)); + if (val == raxNotFound) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (!string2ll(val,sdslen(val),&ll)) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (out_err) *out_err = REDISMODULE_OK; + return ll; +} + +/* Get the value of a field from data collected with RM_GetServerInfo(). If the + * field is not found, or is not numerical or out of range, return value will be + * 0, and the optional out_err argument will be set to REDISMODULE_ERR. */ +unsigned long long RM_ServerInfoGetFieldUnsigned(RedisModuleServerInfoData *data, const char* field, int *out_err) { + unsigned long long ll; + sds val = raxFind(data->rax, (unsigned char *)field, strlen(field)); + if (val == raxNotFound) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (!string2ull(val,&ll)) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (out_err) *out_err = REDISMODULE_OK; + return ll; +} + +/* Get the value of a field from data collected with RM_GetServerInfo(). If the + * field is not found, or is not a double, return value will be 0, and the + * optional out_err argument will be set to REDISMODULE_ERR. */ +double RM_ServerInfoGetFieldDouble(RedisModuleServerInfoData *data, const char* field, int *out_err) { + double dbl; + sds val = raxFind(data->rax, (unsigned char *)field, strlen(field)); + if (val == raxNotFound) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (!string2d(val,sdslen(val),&dbl)) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (out_err) *out_err = REDISMODULE_OK; + return dbl; +} + +/* -------------------------------------------------------------------------- + * Modules utility APIs + * -------------------------------------------------------------------------- */ + +/* Return random bytes using SHA1 in counter mode with a /dev/urandom + * initialized seed. This function is fast so can be used to generate + * many bytes without any effect on the operating system entropy pool. + * Currently this function is not thread safe. */ +void RM_GetRandomBytes(unsigned char *dst, size_t len) { + getRandomBytes(dst,len); +} + +/* Like RedisModule_GetRandomBytes() but instead of setting the string to + * random bytes the string is set to random characters in the in the + * hex charset [0-9a-f]. */ +void RM_GetRandomHexChars(char *dst, size_t len) { + getRandomHexChars(dst,len); +} + +/* -------------------------------------------------------------------------- + * Modules API exporting / importing + * -------------------------------------------------------------------------- */ + +/* This function is called by a module in order to export some API with a + * given name. Other modules will be able to use this API by calling the + * symmetrical function RM_GetSharedAPI() and casting the return value to + * the right function pointer. + * + * The function will return REDISMODULE_OK if the name is not already taken, + * otherwise REDISMODULE_ERR will be returned and no operation will be + * performed. + * + * IMPORTANT: the apiname argument should be a string literal with static + * lifetime. The API relies on the fact that it will always be valid in + * the future. */ +int RM_ExportSharedAPI(RedisModuleCtx *ctx, const char *apiname, void *func) { + RedisModuleSharedAPI *sapi = zmalloc(sizeof(*sapi)); + sapi->module = ctx->module; + sapi->func = func; + if (dictAdd(server.sharedapi, (char*)apiname, sapi) != DICT_OK) { + zfree(sapi); + return REDISMODULE_ERR; + } + return REDISMODULE_OK; +} + +/* Request an exported API pointer. The return value is just a void pointer + * that the caller of this function will be required to cast to the right + * function pointer, so this is a private contract between modules. + * + * If the requested API is not available then NULL is returned. Because + * modules can be loaded at different times with different order, this + * function calls should be put inside some module generic API registering + * step, that is called every time a module attempts to execute a + * command that requires external APIs: if some API cannot be resolved, the + * command should return an error. + * + * Here is an exmaple: + * + * int ... myCommandImplementation() { + * if (getExternalAPIs() == 0) { + * reply with an error here if we cannot have the APIs + * } + * // Use the API: + * myFunctionPointer(foo); + * } + * + * And the function registerAPI() is: + * + * int getExternalAPIs(void) { + * static int api_loaded = 0; + * if (api_loaded != 0) return 1; // APIs already resolved. + * + * myFunctionPointer = RedisModule_GetOtherModuleAPI("..."); + * if (myFunctionPointer == NULL) return 0; + * + * return 1; + * } + */ +void *RM_GetSharedAPI(RedisModuleCtx *ctx, const char *apiname) { dictEntry *de = dictFind(server.sharedapi, apiname); if (de == NULL) return NULL; RedisModuleSharedAPI *sapi = dictGetVal(de); @@ -4854,7 +6237,7 @@ int moduleUnregisterUsedAPI(RedisModule *module) { RedisModule *used = ln->value; listNode *ln = listSearchKey(used->usedby,module); if (ln) { - listDelNode(module->using,ln); + listDelNode(used->usedby,ln); count++; } } @@ -4961,11 +6344,13 @@ int RM_UnregisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilter *fi ln = listSearchKey(moduleCommandFilters,filter); if (!ln) return REDISMODULE_ERR; listDelNode(moduleCommandFilters,ln); - + ln = listSearchKey(ctx->module->filters,filter); if (!ln) return REDISMODULE_ERR; /* Shouldn't happen */ listDelNode(ctx->module->filters,ln); + zfree(filter); + return REDISMODULE_OK; } @@ -5069,6 +6454,749 @@ int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos) return REDISMODULE_OK; } +/* For a given pointer allocated via RedisModule_Alloc() or + * RedisModule_Realloc(), return the amount of memory allocated for it. + * Note that this may be different (larger) than the memory we allocated + * with the allocation calls, since sometimes the underlying allocator + * will allocate more memory. + */ +size_t RM_MallocSize(void* ptr){ + return zmalloc_size(ptr); +} + +/* Return the a number between 0 to 1 indicating the amount of memory + * currently used, relative to the Redis "maxmemory" configuration. + * + * 0 - No memory limit configured. + * Between 0 and 1 - The percentage of the memory used normalized in 0-1 range. + * Exactly 1 - Memory limit reached. + * Greater 1 - More memory used than the configured limit. + */ +float RM_GetUsedMemoryRatio(){ + float level; + getMaxmemoryState(NULL, NULL, NULL, &level); + return level; +} + +/* -------------------------------------------------------------------------- + * Scanning keyspace and hashes + * -------------------------------------------------------------------------- */ + +typedef void (*RedisModuleScanCB)(RedisModuleCtx *ctx, RedisModuleString *keyname, RedisModuleKey *key, void *privdata); +typedef struct { + RedisModuleCtx *ctx; + void* user_data; + RedisModuleScanCB fn; +} ScanCBData; + +typedef struct RedisModuleScanCursor{ + int cursor; + int done; +}RedisModuleScanCursor; + +static void moduleScanCallback(void *privdata, const dictEntry *de) { + ScanCBData *data = privdata; + sds key = dictGetKey(de); + robj* val = dictGetVal(de); + RedisModuleString *keyname = createObject(OBJ_STRING,sdsdup(key)); + + /* Setup the key handle. */ + RedisModuleKey kp = {0}; + moduleInitKey(&kp, data->ctx, keyname, val, REDISMODULE_READ); + + data->fn(data->ctx, keyname, &kp, data->user_data); + + moduleCloseKey(&kp); + decrRefCount(keyname); +} + +/* Create a new cursor to be used with RedisModule_Scan */ +RedisModuleScanCursor *RM_ScanCursorCreate() { + RedisModuleScanCursor* cursor = zmalloc(sizeof(*cursor)); + cursor->cursor = 0; + cursor->done = 0; + return cursor; +} + +/* Restart an existing cursor. The keys will be rescanned. */ +void RM_ScanCursorRestart(RedisModuleScanCursor *cursor) { + cursor->cursor = 0; + cursor->done = 0; +} + +/* Destroy the cursor struct. */ +void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) { + zfree(cursor); +} + +/* Scan API that allows a module to scan all the keys and value in + * the selected db. + * + * Callback for scan implementation. + * void scan_callback(RedisModuleCtx *ctx, RedisModuleString *keyname, + * RedisModuleKey *key, void *privdata); + * ctx - the redis module context provided to for the scan. + * keyname - owned by the caller and need to be retained if used after this + * function. + * + * key - holds info on the key and value, it is provided as best effort, in + * some cases it might be NULL, in which case the user should (can) use + * RedisModule_OpenKey (and CloseKey too). + * when it is provided, it is owned by the caller and will be free when the + * callback returns. + * + * privdata - the user data provided to RedisModule_Scan. + * + * The way it should be used: + * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); + * while(RedisModule_Scan(ctx, c, callback, privateData)); + * RedisModule_ScanCursorDestroy(c); + * + * It is also possible to use this API from another thread while the lock + * is acquired durring the actuall call to RM_Scan: + * + * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); + * RedisModule_ThreadSafeContextLock(ctx); + * while(RedisModule_Scan(ctx, c, callback, privateData)){ + * RedisModule_ThreadSafeContextUnlock(ctx); + * // do some background job + * RedisModule_ThreadSafeContextLock(ctx); + * } + * RedisModule_ScanCursorDestroy(c); + * + * The function will return 1 if there are more elements to scan and + * 0 otherwise, possibly setting errno if the call failed. + * + * It is also possible to restart and existing cursor using RM_CursorRestart. + * + * IMPORTANT: This API is very similar to the Redis SCAN command from the + * point of view of the guarantees it provides. This means that the API + * may report duplicated keys, but guarantees to report at least one time + * every key that was there from the start to the end of the scanning process. + * + * NOTE: If you do database changes within the callback, you should be aware + * that the internal state of the database may change. For instance it is safe + * to delete or modify the current key, but may not be safe to delete any + * other key. + * Moreover playing with the Redis keyspace while iterating may have the + * effect of returning more duplicates. A safe pattern is to store the keys + * names you want to modify elsewhere, and perform the actions on the keys + * later when the iteration is complete. Howerver this can cost a lot of + * memory, so it may make sense to just operate on the current key when + * possible during the iteration, given that this is safe. */ +int RM_Scan(RedisModuleCtx *ctx, RedisModuleScanCursor *cursor, RedisModuleScanCB fn, void *privdata) { + if (cursor->done) { + errno = ENOENT; + return 0; + } + int ret = 1; + ScanCBData data = { ctx, privdata, fn }; + cursor->cursor = dictScan(ctx->client->db->dict, cursor->cursor, moduleScanCallback, NULL, &data); + if (cursor->cursor == 0) { + cursor->done = 1; + ret = 0; + } + errno = 0; + return ret; +} + +typedef void (*RedisModuleScanKeyCB)(RedisModuleKey *key, RedisModuleString *field, RedisModuleString *value, void *privdata); +typedef struct { + RedisModuleKey *key; + void* user_data; + RedisModuleScanKeyCB fn; +} ScanKeyCBData; + +static void moduleScanKeyCallback(void *privdata, const dictEntry *de) { + ScanKeyCBData *data = privdata; + sds key = dictGetKey(de); + robj *o = data->key->value; + robj *field = createStringObject(key, sdslen(key)); + robj *value = NULL; + if (o->type == OBJ_SET) { + value = NULL; + } else if (o->type == OBJ_HASH) { + sds val = dictGetVal(de); + value = createStringObject(val, sdslen(val)); + } else if (o->type == OBJ_ZSET) { + double *val = (double*)dictGetVal(de); + value = createStringObjectFromLongDouble(*val, 0); + } + + data->fn(data->key, field, value, data->user_data); + decrRefCount(field); + if (value) decrRefCount(value); +} + +/* Scan api that allows a module to scan the elements in a hash, set or sorted set key + * + * Callback for scan implementation. + * void scan_callback(RedisModuleKey *key, RedisModuleString* field, RedisModuleString* value, void *privdata); + * - key - the redis key context provided to for the scan. + * - field - field name, owned by the caller and need to be retained if used + * after this function. + * - value - value string or NULL for set type, owned by the caller and need to + * be retained if used after this function. + * - privdata - the user data provided to RedisModule_ScanKey. + * + * The way it should be used: + * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); + * RedisModuleKey *key = RedisModule_OpenKey(...) + * while(RedisModule_ScanKey(key, c, callback, privateData)); + * RedisModule_CloseKey(key); + * RedisModule_ScanCursorDestroy(c); + * + * It is also possible to use this API from another thread while the lock is acquired durring + * the actuall call to RM_Scan, and re-opening the key each time: + * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); + * RedisModule_ThreadSafeContextLock(ctx); + * RedisModuleKey *key = RedisModule_OpenKey(...) + * while(RedisModule_ScanKey(ctx, c, callback, privateData)){ + * RedisModule_CloseKey(key); + * RedisModule_ThreadSafeContextUnlock(ctx); + * // do some background job + * RedisModule_ThreadSafeContextLock(ctx); + * RedisModuleKey *key = RedisModule_OpenKey(...) + * } + * RedisModule_CloseKey(key); + * RedisModule_ScanCursorDestroy(c); + * + * The function will return 1 if there are more elements to scan and 0 otherwise, + * possibly setting errno if the call failed. + * It is also possible to restart and existing cursor using RM_CursorRestart. + * + * NOTE: Certain operations are unsafe while iterating the object. For instance + * while the API guarantees to return at least one time all the elements that + * are present in the data structure consistently from the start to the end + * of the iteration (see HSCAN and similar commands documentation), the more + * you play with the elements, the more duplicates you may get. In general + * deleting the current element of the data structure is safe, while removing + * the key you are iterating is not safe. */ +int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleScanKeyCB fn, void *privdata) { + if (key == NULL || key->value == NULL) { + errno = EINVAL; + return 0; + } + dict *ht = NULL; + robj *o = key->value; + if (o->type == OBJ_SET) { + if (o->encoding == OBJ_ENCODING_HT) + ht = o->ptr; + } else if (o->type == OBJ_HASH) { + if (o->encoding == OBJ_ENCODING_HT) + ht = o->ptr; + } else if (o->type == OBJ_ZSET) { + if (o->encoding == OBJ_ENCODING_SKIPLIST) + ht = ((zset *)o->ptr)->dict; + } else { + errno = EINVAL; + return 0; + } + if (cursor->done) { + errno = ENOENT; + return 0; + } + int ret = 1; + if (ht) { + ScanKeyCBData data = { key, privdata, fn }; + cursor->cursor = dictScan(ht, cursor->cursor, moduleScanKeyCallback, NULL, &data); + if (cursor->cursor == 0) { + cursor->done = 1; + ret = 0; + } + } else if (o->type == OBJ_SET && o->encoding == OBJ_ENCODING_INTSET) { + int pos = 0; + int64_t ll; + while(intsetGet(o->ptr,pos++,&ll)) { + robj *field = createStringObjectFromLongLong(ll); + fn(key, field, NULL, privdata); + decrRefCount(field); + } + cursor->cursor = 1; + cursor->done = 1; + ret = 0; + } else if (o->type == OBJ_HASH || o->type == OBJ_ZSET) { + unsigned char *p = ziplistIndex(o->ptr,0); + unsigned char *vstr; + unsigned int vlen; + long long vll; + while(p) { + ziplistGet(p,&vstr,&vlen,&vll); + robj *field = (vstr != NULL) ? + createStringObject((char*)vstr,vlen) : + createStringObjectFromLongLong(vll); + p = ziplistNext(o->ptr,p); + ziplistGet(p,&vstr,&vlen,&vll); + robj *value = (vstr != NULL) ? + createStringObject((char*)vstr,vlen) : + createStringObjectFromLongLong(vll); + fn(key, field, value, privdata); + p = ziplistNext(o->ptr,p); + decrRefCount(field); + decrRefCount(value); + } + cursor->cursor = 1; + cursor->done = 1; + ret = 0; + } + errno = 0; + return ret; +} + + +/* -------------------------------------------------------------------------- + * Module fork API + * -------------------------------------------------------------------------- */ + +/* Create a background child process with the current frozen snaphost of the + * main process where you can do some processing in the background without + * affecting / freezing the traffic and no need for threads and GIL locking. + * Note that Redis allows for only one concurrent fork. + * When the child wants to exit, it should call RedisModule_ExitFromChild. + * If the parent wants to kill the child it should call RedisModule_KillForkChild + * The done handler callback will be executed on the parent process when the + * child existed (but not when killed) + * Return: -1 on failure, on success the parent process will get a positive PID + * of the child, and the child process will get 0. + */ +int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) { + pid_t childpid; + if (hasActiveChildProcess()) { + return -1; + } + + openChildInfoPipe(); + if ((childpid = redisFork()) == 0) { + /* Child */ + redisSetProcTitle("redis-module-fork"); + } else if (childpid == -1) { + closeChildInfoPipe(); + serverLog(LL_WARNING,"Can't fork for module: %s", strerror(errno)); + } else { + /* Parent */ + server.module_child_pid = childpid; + moduleForkInfo.done_handler = cb; + moduleForkInfo.done_handler_user_data = user_data; + serverLog(LL_VERBOSE, "Module fork started pid: %d ", childpid); + } + return childpid; +} + +/* Call from the child process when you want to terminate it. + * retcode will be provided to the done handler executed on the parent process. + */ +int RM_ExitFromChild(int retcode) { + sendChildCOWInfo(CHILD_INFO_TYPE_MODULE, "Module fork"); + exitFromChild(retcode); + return REDISMODULE_OK; +} + +/* Kill the active module forked child, if there is one active and the + * pid matches, and returns C_OK. Otherwise if there is no active module + * child or the pid does not match, return C_ERR without doing anything. */ +int TerminateModuleForkChild(int child_pid, int wait) { + /* Module child should be active and pid should match. */ + if (server.module_child_pid == -1 || + server.module_child_pid != child_pid) return C_ERR; + + int statloc; + serverLog(LL_VERBOSE,"Killing running module fork child: %ld", + (long) server.module_child_pid); + if (kill(server.module_child_pid,SIGUSR1) != -1 && wait) { + while(wait4(server.module_child_pid,&statloc,0,NULL) != + server.module_child_pid); + } + /* Reset the buffer accumulating changes while the child saves. */ + server.module_child_pid = -1; + moduleForkInfo.done_handler = NULL; + moduleForkInfo.done_handler_user_data = NULL; + closeChildInfoPipe(); + updateDictResizePolicy(); + return C_OK; +} + +/* Can be used to kill the forked child process from the parent process. + * child_pid whould be the return value of RedisModule_Fork. */ +int RM_KillForkChild(int child_pid) { + /* Kill module child, wait for child exit. */ + if (TerminateModuleForkChild(child_pid,1) == C_OK) + return REDISMODULE_OK; + else + return REDISMODULE_ERR; +} + +void ModuleForkDoneHandler(int exitcode, int bysignal) { + serverLog(LL_NOTICE, + "Module fork exited pid: %d, retcode: %d, bysignal: %d", + server.module_child_pid, exitcode, bysignal); + if (moduleForkInfo.done_handler) { + moduleForkInfo.done_handler(exitcode, bysignal, + moduleForkInfo.done_handler_user_data); + } + server.module_child_pid = -1; + moduleForkInfo.done_handler = NULL; + moduleForkInfo.done_handler_user_data = NULL; +} + +/* -------------------------------------------------------------------------- + * Server hooks implementation + * -------------------------------------------------------------------------- */ + +/* Register to be notified, via a callback, when the specified server event + * happens. The callback is called with the event as argument, and an additional + * argument which is a void pointer and should be cased to a specific type + * that is event-specific (but many events will just use NULL since they do not + * have additional information to pass to the callback). + * + * If the callback is NULL and there was a previous subscription, the module + * will be unsubscribed. If there was a previous subscription and the callback + * is not null, the old callback will be replaced with the new one. + * + * The callback must be of this type: + * + * int (*RedisModuleEventCallback)(RedisModuleCtx *ctx, + * RedisModuleEvent eid, + * uint64_t subevent, + * void *data); + * + * The 'ctx' is a normal Redis module context that the callback can use in + * order to call other modules APIs. The 'eid' is the event itself, this + * is only useful in the case the module subscribed to multiple events: using + * the 'id' field of this structure it is possible to check if the event + * is one of the events we registered with this callback. The 'subevent' field + * depends on the event that fired. + * + * Finally the 'data' pointer may be populated, only for certain events, with + * more relevant data. + * + * Here is a list of events you can use as 'eid' and related sub events: + * + * RedisModuleEvent_ReplicationRoleChanged + * + * This event is called when the instance switches from master + * to replica or the other way around, however the event is + * also called when the replica remains a replica but starts to + * replicate with a different master. + * + * The following sub events are available: + * + * REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_MASTER + * REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_REPLICA + * + * The 'data' field can be casted by the callback to a + * RedisModuleReplicationInfo structure with the following fields: + * + * int master; // true if master, false if replica + * char *masterhost; // master instance hostname for NOW_REPLICA + * int masterport; // master instance port for NOW_REPLICA + * char *replid1; // Main replication ID + * char *replid2; // Secondary replication ID + * uint64_t repl1_offset; // Main replication offset + * uint64_t repl2_offset; // Offset of replid2 validity + * + * RedisModuleEvent_Persistence + * + * This event is called when RDB saving or AOF rewriting starts + * and ends. The following sub events are available: + * + * REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START + * REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START + * REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START + * REDISMODULE_SUBEVENT_PERSISTENCE_ENDED + * REDISMODULE_SUBEVENT_PERSISTENCE_FAILED + * + * The above events are triggered not just when the user calls the + * relevant commands like BGSAVE, but also when a saving operation + * or AOF rewriting occurs because of internal server triggers. + * The SYNC_RDB_START sub events are happening in the forground due to + * SAVE command, FLUSHALL, or server shutdown, and the other RDB and + * AOF sub events are executed in a background fork child, so any + * action the module takes can only affect the generated AOF or RDB, + * but will not be reflected in the parent process and affect connected + * clients and commands. Also note that the AOF_START sub event may end + * up saving RDB content in case of an AOF with rdb-preamble. + * + * RedisModuleEvent_FlushDB + * + * The FLUSHALL, FLUSHDB or an internal flush (for instance + * because of replication, after the replica synchronization) + * happened. The following sub events are available: + * + * REDISMODULE_SUBEVENT_FLUSHDB_START + * REDISMODULE_SUBEVENT_FLUSHDB_END + * + * The data pointer can be casted to a RedisModuleFlushInfo + * structure with the following fields: + * + * int32_t async; // True if the flush is done in a thread. + * See for instance FLUSHALL ASYNC. + * In this case the END callback is invoked + * immediately after the database is put + * in the free list of the thread. + * int32_t dbnum; // Flushed database number, -1 for all the DBs + * in the case of the FLUSHALL operation. + * + * The start event is called *before* the operation is initated, thus + * allowing the callback to call DBSIZE or other operation on the + * yet-to-free keyspace. + * + * RedisModuleEvent_Loading + * + * Called on loading operations: at startup when the server is + * started, but also after a first synchronization when the + * replica is loading the RDB file from the master. + * The following sub events are available: + * + * REDISMODULE_SUBEVENT_LOADING_RDB_START + * REDISMODULE_SUBEVENT_LOADING_AOF_START + * REDISMODULE_SUBEVENT_LOADING_REPL_START + * REDISMODULE_SUBEVENT_LOADING_ENDED + * REDISMODULE_SUBEVENT_LOADING_FAILED + * + * Note that AOF loading may start with an RDB data in case of + * rdb-preamble, in which case you'll only recieve an AOF_START event. + * + * + * RedisModuleEvent_ClientChange + * + * Called when a client connects or disconnects. + * The data pointer can be casted to a RedisModuleClientInfo + * structure, documented in RedisModule_GetClientInfoById(). + * The following sub events are available: + * + * REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED + * REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED + * + * RedisModuleEvent_Shutdown + * + * The server is shutting down. No subevents are available. + * + * RedisModuleEvent_ReplicaChange + * + * This event is called when the instance (that can be both a + * master or a replica) get a new online replica, or lose a + * replica since it gets disconnected. + * The following sub events are availble: + * + * REDISMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE + * REDISMODULE_SUBEVENT_REPLICA_CHANGE_OFFLINE + * + * No additional information is available so far: future versions + * of Redis will have an API in order to enumerate the replicas + * connected and their state. + * + * RedisModuleEvent_CronLoop + * + * This event is called every time Redis calls the serverCron() + * function in order to do certain bookkeeping. Modules that are + * required to do operations from time to time may use this callback. + * Normally Redis calls this function 10 times per second, but + * this changes depending on the "hz" configuration. + * No sub events are available. + * + * The data pointer can be casted to a RedisModuleCronLoop + * structure with the following fields: + * + * int32_t hz; // Approximate number of events per second. + * + * RedisModuleEvent_MasterLinkChange + * + * This is called for replicas in order to notify when the + * replication link becomes functional (up) with our master, + * or when it goes down. Note that the link is not considered + * up when we just connected to the master, but only if the + * replication is happening correctly. + * The following sub events are available: + * + * REDISMODULE_SUBEVENT_MASTER_LINK_UP + * REDISMODULE_SUBEVENT_MASTER_LINK_DOWN + * + * RedisModuleEvent_ModuleChange + * + * This event is called when a new module is loaded or one is unloaded. + * The following sub events are availble: + * + * REDISMODULE_SUBEVENT_MODULE_LOADED + * REDISMODULE_SUBEVENT_MODULE_UNLOADED + * + * The data pointer can be casted to a RedisModuleModuleChange + * structure with the following fields: + * + * const char* module_name; // Name of module loaded or unloaded. + * int32_t module_version; // Module version. + * + * RedisModuleEvent_LoadingProgress + * + * This event is called repeatedly called while an RDB or AOF file + * is being loaded. + * The following sub events are availble: + * + * REDISMODULE_SUBEVENT_LOADING_PROGRESS_RDB + * REDISMODULE_SUBEVENT_LOADING_PROGRESS_AOF + * + * The data pointer can be casted to a RedisModuleLoadingProgress + * structure with the following fields: + * + * int32_t hz; // Approximate number of events per second. + * int32_t progress; // Approximate progress between 0 and 1024, + * or -1 if unknown. + * + * The function returns REDISMODULE_OK if the module was successfully subscrived + * for the specified event. If the API is called from a wrong context then + * REDISMODULE_ERR is returned. */ +int RM_SubscribeToServerEvent(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback) { + RedisModuleEventListener *el; + + /* Protect in case of calls from contexts without a module reference. */ + if (ctx->module == NULL) return REDISMODULE_ERR; + + /* Search an event matching this module and event ID. */ + listIter li; + listNode *ln; + listRewind(RedisModule_EventListeners,&li); + while((ln = listNext(&li))) { + el = ln->value; + if (el->module == ctx->module && el->event.id == event.id) + break; /* Matching event found. */ + } + + /* Modify or remove the event listener if we already had one. */ + if (ln) { + if (callback == NULL) { + listDelNode(RedisModule_EventListeners,ln); + zfree(el); + } else { + el->callback = callback; /* Update the callback with the new one. */ + } + return REDISMODULE_OK; + } + + /* No event found, we need to add a new one. */ + el = zmalloc(sizeof(*el)); + el->module = ctx->module; + el->event = event; + el->callback = callback; + listAddNodeTail(RedisModule_EventListeners,el); + return REDISMODULE_OK; +} + +/* This is called by the Redis internals every time we want to fire an + * event that can be interceppted by some module. The pointer 'data' is useful + * in order to populate the event-specific structure when needed, in order + * to return the structure with more information to the callback. + * + * 'eid' and 'subid' are just the main event ID and the sub event associated + * with the event, depending on what exactly happened. */ +void moduleFireServerEvent(uint64_t eid, int subid, void *data) { + /* Fast path to return ASAP if there is nothing to do, avoiding to + * setup the iterator and so forth: we want this call to be extremely + * cheap if there are no registered modules. */ + if (listLength(RedisModule_EventListeners) == 0) return; + + listIter li; + listNode *ln; + listRewind(RedisModule_EventListeners,&li); + while((ln = listNext(&li))) { + RedisModuleEventListener *el = ln->value; + if (el->event.id == eid) { + RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.module = el->module; + + if (ModulesInHooks == 0) { + ctx.client = moduleFreeContextReusedClient; + } else { + ctx.client = createClient(NULL); + ctx.client->flags |= CLIENT_MODULE; + ctx.client->user = NULL; /* Root user. */ + } + + void *moduledata = NULL; + RedisModuleClientInfoV1 civ1; + RedisModuleReplicationInfoV1 riv1; + RedisModuleModuleChangeV1 mcv1; + /* Start at DB zero by default when calling the handler. It's + * up to the specific event setup to change it when it makes + * sense. For instance for FLUSHDB events we select the correct + * DB automatically. */ + selectDb(ctx.client, 0); + + /* Event specific context and data pointer setup. */ + if (eid == REDISMODULE_EVENT_CLIENT_CHANGE) { + modulePopulateClientInfoStructure(&civ1,data, + el->event.dataver); + moduledata = &civ1; + } else if (eid == REDISMODULE_EVENT_REPLICATION_ROLE_CHANGED) { + modulePopulateReplicationInfoStructure(&riv1,el->event.dataver); + moduledata = &riv1; + } else if (eid == REDISMODULE_EVENT_FLUSHDB) { + moduledata = data; + RedisModuleFlushInfoV1 *fi = data; + if (fi->dbnum != -1) + selectDb(ctx.client, fi->dbnum); + } else if (eid == REDISMODULE_EVENT_MODULE_CHANGE) { + RedisModule *m = data; + if (m == el->module) + continue; + mcv1.version = REDISMODULE_MODULE_CHANGE_VERSION; + mcv1.module_name = m->name; + mcv1.module_version = m->ver; + moduledata = &mcv1; + } else if (eid == REDISMODULE_EVENT_LOADING_PROGRESS) { + moduledata = data; + } else if (eid == REDISMODULE_EVENT_CRON_LOOP) { + moduledata = data; + } + + ModulesInHooks++; + el->module->in_hook++; + el->callback(&ctx,el->event,subid,moduledata); + el->module->in_hook--; + ModulesInHooks--; + + if (ModulesInHooks != 0) freeClient(ctx.client); + moduleFreeContext(&ctx); + } + } +} + +/* Remove all the listeners for this module: this is used before unloading + * a module. */ +void moduleUnsubscribeAllServerEvents(RedisModule *module) { + RedisModuleEventListener *el; + listIter li; + listNode *ln; + listRewind(RedisModule_EventListeners,&li); + + while((ln = listNext(&li))) { + el = ln->value; + if (el->module == module) { + listDelNode(RedisModule_EventListeners,ln); + zfree(el); + } + } +} + +void processModuleLoadingProgressEvent(int is_aof) { + long long now = ustime(); + static long long next_event = 0; + if (now >= next_event) { + /* Fire the loading progress modules end event. */ + int progress = -1; + if (server.loading_total_bytes) + progress = (server.loading_total_bytes<<10) / server.loading_total_bytes; + RedisModuleFlushInfoV1 fi = {REDISMODULE_LOADING_PROGRESS_VERSION, + server.hz, + progress}; + moduleFireServerEvent(REDISMODULE_EVENT_LOADING_PROGRESS, + is_aof? + REDISMODULE_SUBEVENT_LOADING_PROGRESS_AOF: + REDISMODULE_SUBEVENT_LOADING_PROGRESS_RDB, + &fi); + /* decide when the next event should fire. */ + next_event = now + 1000000 / server.hz; + } +} + /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ @@ -5111,8 +7239,9 @@ void moduleInitModulesSystem(void) { /* Set up the keyspace notification susbscriber list and static client */ moduleKeyspaceSubscribers = listCreate(); - moduleFreeContextReusedClient = createClient(-1); + moduleFreeContextReusedClient = createClient(NULL); moduleFreeContextReusedClient->flags |= CLIENT_MODULE; + moduleFreeContextReusedClient->user = NULL; /* root user. */ /* Set up filter list */ moduleCommandFilters = listCreate(); @@ -5132,6 +7261,9 @@ void moduleInitModulesSystem(void) { /* Create the timers radix tree. */ Timers = raxNew(); + /* Setup the event listeners data structures. */ + RedisModule_EventListeners = listCreate(); + /* Our thread-safe contexts GIL must start with already locked: * it is just unlocked when it's safe. */ pthread_mutex_lock(&moduleGIL); @@ -5167,6 +7299,8 @@ void moduleLoadFromQueue(void) { void moduleFreeModuleStructure(struct RedisModule *module) { listRelease(module->types); listRelease(module->filters); + listRelease(module->usedby); + listRelease(module->using); sdsfree(module->name); zfree(module); } @@ -5199,6 +7333,17 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) { int (*onload)(void *, void **, int); void *handle; RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.client = moduleFreeContextReusedClient; + selectDb(ctx.client, 0); + + struct stat st; + if (stat(path, &st) == 0) + { // this check is best effort + if (!(st.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH))) { + serverLog(LL_WARNING, "Module %s failed to load: It does not have execute permissions.", path); + return C_ERR; + } + } handle = dlopen(path,RTLD_NOW|RTLD_LOCAL); if (handle == NULL) { @@ -5228,8 +7373,14 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) { /* Redis module loaded! Register it. */ dictAdd(modules,ctx.module->name,ctx.module); + ctx.module->blocked_clients = 0; ctx.module->handle = handle; serverLog(LL_NOTICE,"Module '%s' loaded from %s",ctx.module->name,path); + /* Fire the loaded modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_MODULE_CHANGE, + REDISMODULE_SUBEVENT_MODULE_LOADED, + ctx.module); + moduleFreeContext(&ctx); return C_OK; } @@ -5253,8 +7404,29 @@ int moduleUnload(sds name) { } else if (listLength(module->usedby)) { errno = EPERM; return REDISMODULE_ERR; + } else if (module->blocked_clients) { + errno = EAGAIN; + return REDISMODULE_ERR; } + /* Give module a chance to clean up. */ + int (*onunload)(void *); + onunload = (int (*)(void *))(unsigned long) dlsym(module->handle, "RedisModule_OnUnload"); + if (onunload) { + RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.module = module; + ctx.client = moduleFreeContextReusedClient; + int unload_status = onunload((void*)&ctx); + moduleFreeContext(&ctx); + + if (unload_status == REDISMODULE_ERR) { + serverLog(LL_WARNING, "Module %s OnUnload failed. Unload canceled.", name); + errno = ECANCELED; + return REDISMODULE_ERR; + } + } + + moduleFreeAuthenticatedClients(module); moduleUnregisterCommands(module); moduleUnregisterSharedAPI(module); moduleUnregisterUsedAPI(module); @@ -5262,8 +7434,7 @@ int moduleUnload(sds name) { /* Remove any notification subscribers this module might have */ moduleUnsubscribeNotifications(module); - - /* Unregister all the hooks. TODO: Yet no hooks support here. */ + moduleUnsubscribeAllServerEvents(module); /* Unload the dynamic library. */ if (dlclose(module->handle) == -1) { @@ -5273,6 +7444,11 @@ int moduleUnload(sds name) { module->name, error); } + /* Fire the unloaded modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_MODULE_CHANGE, + REDISMODULE_SUBEVENT_MODULE_UNLOADED, + module); + /* Remove from list of modules. */ serverLog(LL_NOTICE,"Module %s unloaded",module->name); dictDelete(modules,module->name); @@ -5282,6 +7458,81 @@ int moduleUnload(sds name) { return REDISMODULE_OK; } +/* Helper function for the MODULE and HELLO command: send the list of the + * loaded modules to the client. */ +void addReplyLoadedModules(client *c) { + dictIterator *di = dictGetIterator(modules); + dictEntry *de; + + addReplyArrayLen(c,dictSize(modules)); + while ((de = dictNext(di)) != NULL) { + sds name = dictGetKey(de); + struct RedisModule *module = dictGetVal(de); + addReplyMapLen(c,2); + addReplyBulkCString(c,"name"); + addReplyBulkCBuffer(c,name,sdslen(name)); + addReplyBulkCString(c,"ver"); + addReplyLongLong(c,module->ver); + } + dictReleaseIterator(di); +} + +/* Helper for genModulesInfoString(): given a list of modules, return + * am SDS string in the form "[modulename|modulename2|...]" */ +sds genModulesInfoStringRenderModulesList(list *l) { + listIter li; + listNode *ln; + listRewind(l,&li); + sds output = sdsnew("["); + while((ln = listNext(&li))) { + RedisModule *module = ln->value; + output = sdscat(output,module->name); + } + output = sdstrim(output,"|"); + output = sdscat(output,"]"); + return output; +} + +/* Helper for genModulesInfoString(): render module options as an SDS string. */ +sds genModulesInfoStringRenderModuleOptions(struct RedisModule *module) { + sds output = sdsnew("["); + if (module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) + output = sdscat(output,"handle-io-errors|"); + output = sdstrim(output,"|"); + output = sdscat(output,"]"); + return output; +} + + +/* Helper function for the INFO command: adds loaded modules as to info's + * output. + * + * After the call, the passed sds info string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds genModulesInfoString(sds info) { + dictIterator *di = dictGetIterator(modules); + dictEntry *de; + + while ((de = dictNext(di)) != NULL) { + sds name = dictGetKey(de); + struct RedisModule *module = dictGetVal(de); + + sds usedby = genModulesInfoStringRenderModulesList(module->usedby); + sds using = genModulesInfoStringRenderModulesList(module->using); + sds options = genModulesInfoStringRenderModuleOptions(module); + info = sdscatfmt(info, + "module:name=%S,ver=%i,api=%i,filters=%i," + "usedby=%S,using=%S,options=%S\r\n", + name, module->ver, module->apiver, + (int)listLength(module->filters), usedby, using, options); + sdsfree(usedby); + sdsfree(using); + sdsfree(options); + } + dictReleaseIterator(di); + return info; +} + /* Redis MODULE command. * * MODULE LOAD [args...] */ @@ -5327,6 +7578,10 @@ NULL errmsg = "the module exports APIs used by other modules. " "Please unload them first and try again"; break; + case EAGAIN: + errmsg = "the module has blocked clients. " + "Please wait them unblocked and try again"; + break; default: errmsg = "operation not possible."; break; @@ -5334,20 +7589,7 @@ NULL addReplyErrorFormat(c,"Error unloading module: %s",errmsg); } } else if (!strcasecmp(subcmd,"list") && c->argc == 2) { - dictIterator *di = dictGetIterator(modules); - dictEntry *de; - - addReplyMultiBulkLen(c,dictSize(modules)); - while ((de = dictNext(di)) != NULL) { - sds name = dictGetKey(de); - struct RedisModule *module = dictGetVal(de); - addReplyMultiBulkLen(c,4); - addReplyBulkCString(c,"name"); - addReplyBulkCBuffer(c,name,sdslen(name)); - addReplyBulkCString(c,"ver"); - addReplyLongLong(c,module->ver); - } - dictReleaseIterator(di); + addReplyLoadedModules(c); } else { addReplySubcommandSyntaxError(c); return; @@ -5359,6 +7601,90 @@ size_t moduleCount(void) { return dictSize(modules); } +/* Set the key last access time for LRU based eviction. not relevent if the + * servers's maxmemory policy is LFU based. Value is idle time in milliseconds. + * returns REDISMODULE_OK if the LRU was updated, REDISMODULE_ERR otherwise. */ +int RM_SetLRU(RedisModuleKey *key, mstime_t lru_idle) { + if (!key->value) + return REDISMODULE_ERR; + if (objectSetLRUOrLFU(key->value, -1, lru_idle, lru_idle>=0 ? LRU_CLOCK() : 0, 1)) + return REDISMODULE_OK; + return REDISMODULE_ERR; +} + +/* Gets the key last access time. + * Value is idletime in milliseconds or -1 if the server's eviction policy is + * LFU based. + * returns REDISMODULE_OK if when key is valid. */ +int RM_GetLRU(RedisModuleKey *key, mstime_t *lru_idle) { + *lru_idle = -1; + if (!key->value) + return REDISMODULE_ERR; + if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) + return REDISMODULE_OK; + *lru_idle = estimateObjectIdleTime(key->value); + return REDISMODULE_OK; +} + +/* Set the key access frequency. only relevant if the server's maxmemory policy + * is LFU based. + * The frequency is a logarithmic counter that provides an indication of + * the access frequencyonly (must be <= 255). + * returns REDISMODULE_OK if the LFU was updated, REDISMODULE_ERR otherwise. */ +int RM_SetLFU(RedisModuleKey *key, long long lfu_freq) { + if (!key->value) + return REDISMODULE_ERR; + if (objectSetLRUOrLFU(key->value, lfu_freq, -1, 0, 1)) + return REDISMODULE_OK; + return REDISMODULE_ERR; +} + +/* Gets the key access frequency or -1 if the server's eviction policy is not + * LFU based. + * returns REDISMODULE_OK if when key is valid. */ +int RM_GetLFU(RedisModuleKey *key, long long *lfu_freq) { + *lfu_freq = -1; + if (!key->value) + return REDISMODULE_ERR; + if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) + *lfu_freq = LFUDecrAndReturn(key->value); + return REDISMODULE_OK; +} + +/* Replace the value assigned to a module type. + * + * The key must be open for writing, have an existing value, and have a moduleType + * that matches the one specified by the caller. + * + * Unlike RM_ModuleTypeSetValue() which will free the old value, this function + * simply swaps the old value with the new value. + * + * The function returns REDISMODULE_OK on success, REDISMODULE_ERR on errors + * such as: + * + * 1. Key is not opened for writing. + * 2. Key is not a module data type key. + * 3. Key is a module datatype other than 'mt'. + * + * If old_value is non-NULL, the old value is returned by reference. + */ +int RM_ModuleTypeReplaceValue(RedisModuleKey *key, moduleType *mt, void *new_value, void **old_value) { + if (!(key->mode & REDISMODULE_WRITE) || key->iter) + return REDISMODULE_ERR; + if (!key->value || key->value->type != OBJ_MODULE) + return REDISMODULE_ERR; + + moduleValue *mv = key->value->ptr; + if (mv->type != mt) + return REDISMODULE_ERR; + + if (old_value) + *old_value = mv->value; + mv->value = new_value; + + return REDISMODULE_OK; +} + /* Register all the APIs we export. Keep this function at the end of the * file so that's easy to seek it to add new entries. */ void moduleRegisterCoreAPI(void) { @@ -5377,13 +7703,18 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ReplyWithError); REGISTER_API(ReplyWithSimpleString); REGISTER_API(ReplyWithArray); + REGISTER_API(ReplyWithNullArray); + REGISTER_API(ReplyWithEmptyArray); REGISTER_API(ReplySetArrayLength); REGISTER_API(ReplyWithString); + REGISTER_API(ReplyWithEmptyString); + REGISTER_API(ReplyWithVerbatimString); REGISTER_API(ReplyWithStringBuffer); REGISTER_API(ReplyWithCString); REGISTER_API(ReplyWithNull); REGISTER_API(ReplyWithCallReply); REGISTER_API(ReplyWithDouble); + REGISTER_API(ReplyWithLongDouble); REGISTER_API(GetSelectedDb); REGISTER_API(SelectDb); REGISTER_API(OpenKey); @@ -5394,6 +7725,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ListPop); REGISTER_API(StringToLongLong); REGISTER_API(StringToDouble); + REGISTER_API(StringToLongDouble); REGISTER_API(Call); REGISTER_API(CallReplyProto); REGISTER_API(FreeCallReply); @@ -5405,6 +7737,8 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CreateStringFromCallReply); REGISTER_API(CreateString); REGISTER_API(CreateStringFromLongLong); + REGISTER_API(CreateStringFromDouble); + REGISTER_API(CreateStringFromLongDouble); REGISTER_API(CreateStringFromString); REGISTER_API(CreateStringPrintf); REGISTER_API(FreeString); @@ -5419,6 +7753,9 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(StringTruncate); REGISTER_API(SetExpire); REGISTER_API(GetExpire); + REGISTER_API(ResetDataset); + REGISTER_API(DbSize); + REGISTER_API(RandomKey); REGISTER_API(ZsetAdd); REGISTER_API(ZsetIncrby); REGISTER_API(ZsetScore); @@ -5438,11 +7775,16 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(KeyAtPos); REGISTER_API(GetClientId); REGISTER_API(GetContextFlags); + REGISTER_API(AvoidReplicaTraffic); REGISTER_API(PoolAlloc); REGISTER_API(CreateDataType); REGISTER_API(ModuleTypeSetValue); + REGISTER_API(ModuleTypeReplaceValue); REGISTER_API(ModuleTypeGetType); REGISTER_API(ModuleTypeGetValue); + REGISTER_API(IsIOError); + REGISTER_API(SetModuleOptions); + REGISTER_API(SignalModifiedKey); REGISTER_API(SaveUnsigned); REGISTER_API(LoadUnsigned); REGISTER_API(SaveSigned); @@ -5455,14 +7797,21 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(LoadDouble); REGISTER_API(SaveFloat); REGISTER_API(LoadFloat); + REGISTER_API(SaveLongDouble); + REGISTER_API(LoadLongDouble); + REGISTER_API(SaveDataTypeToString); + REGISTER_API(LoadDataTypeFromString); REGISTER_API(EmitAOF); REGISTER_API(Log); REGISTER_API(LogIOError); + REGISTER_API(_Assert); + REGISTER_API(LatencyAddSample); REGISTER_API(StringAppendBuffer); REGISTER_API(RetainString); REGISTER_API(StringCompare); REGISTER_API(GetContextFromIO); REGISTER_API(GetKeyNameFromIO); + REGISTER_API(GetKeyNameFromModuleKey); REGISTER_API(BlockClient); REGISTER_API(UnblockClient); REGISTER_API(IsBlockedReplyRequest); @@ -5477,6 +7826,8 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(DigestAddStringBuffer); REGISTER_API(DigestAddLongLong); REGISTER_API(DigestEndSequence); + REGISTER_API(NotifyKeyspaceEvent); + REGISTER_API(GetNotifyKeyspaceEvents); REGISTER_API(SubscribeToKeyspaceEvents); REGISTER_API(RegisterClusterMessageReceiver); REGISTER_API(SendClusterMessage); @@ -5525,4 +7876,46 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CommandFilterArgInsert); REGISTER_API(CommandFilterArgReplace); REGISTER_API(CommandFilterArgDelete); + REGISTER_API(Fork); + REGISTER_API(ExitFromChild); + REGISTER_API(KillForkChild); + REGISTER_API(RegisterInfoFunc); + REGISTER_API(InfoAddSection); + REGISTER_API(InfoBeginDictField); + REGISTER_API(InfoEndDictField); + REGISTER_API(InfoAddFieldString); + REGISTER_API(InfoAddFieldCString); + REGISTER_API(InfoAddFieldDouble); + REGISTER_API(InfoAddFieldLongLong); + REGISTER_API(InfoAddFieldULongLong); + REGISTER_API(GetServerInfo); + REGISTER_API(FreeServerInfo); + REGISTER_API(ServerInfoGetField); + REGISTER_API(ServerInfoGetFieldC); + REGISTER_API(ServerInfoGetFieldSigned); + REGISTER_API(ServerInfoGetFieldUnsigned); + REGISTER_API(ServerInfoGetFieldDouble); + REGISTER_API(GetClientInfoById); + REGISTER_API(PublishMessage); + REGISTER_API(SubscribeToServerEvent); + REGISTER_API(SetLRU); + REGISTER_API(GetLRU); + REGISTER_API(SetLFU); + REGISTER_API(GetLFU); + REGISTER_API(BlockClientOnKeys); + REGISTER_API(SignalKeyAsReady); + REGISTER_API(GetBlockedClientReadyKey); + REGISTER_API(GetUsedMemoryRatio); + REGISTER_API(MallocSize); + REGISTER_API(ScanCursorCreate); + REGISTER_API(ScanCursorDestroy); + REGISTER_API(ScanCursorRestart); + REGISTER_API(Scan); + REGISTER_API(ScanKey); + REGISTER_API(CreateModuleUser); + REGISTER_API(SetModuleUserACL); + REGISTER_API(FreeModuleUser); + REGISTER_API(DeauthenticateAndCloseClient); + REGISTER_API(AuthenticateClientWithACLUser); + REGISTER_API(AuthenticateClientWithUser); } diff --git a/redis.submodule/src/modules/Makefile b/redis.submodule/src/modules/Makefile index 4f6b50f..5e012d6 100644 --- a/redis.submodule/src/modules/Makefile +++ b/redis.submodule/src/modules/Makefile @@ -13,7 +13,7 @@ endif .SUFFIXES: .c .so .xo .o -all: helloworld.so hellotype.so helloblock.so testmodule.so hellocluster.so hellotimer.so hellodict.so +all: helloworld.so hellotype.so helloblock.so testmodule.so hellocluster.so hellotimer.so hellodict.so hellohook.so helloacl.so .c.xo: $(CC) -I. $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ @@ -46,6 +46,17 @@ hellotimer.so: hellotimer.xo hellodict.xo: ../redismodule.h hellodict.so: hellodict.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc + +hellohook.xo: ../redismodule.h + +hellohook.so: hellohook.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc + +helloacl.xo: ../redismodule.h + +helloacl.so: helloacl.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc testmodule.xo: ../redismodule.h diff --git a/redis.submodule/src/modules/helloacl.c b/redis.submodule/src/modules/helloacl.c new file mode 100644 index 0000000..6766c0a --- /dev/null +++ b/redis.submodule/src/modules/helloacl.c @@ -0,0 +1,191 @@ +/* ACL API example - An example for performing custom synchronous and + * asynchronous password authentication. + * + * ----------------------------------------------------------------------------- + * + * Copyright 2019 Amazon.com, Inc. or its affiliates. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define REDISMODULE_EXPERIMENTAL_API +#include "../redismodule.h" +#include +#include + +// A simple global user +static RedisModuleUser *global; +static uint64_t global_auth_client_id = 0; + +/* HELLOACL.REVOKE + * Synchronously revoke access from a user. */ +int RevokeCommand_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (global_auth_client_id) { + RedisModule_DeauthenticateAndCloseClient(ctx, global_auth_client_id); + return RedisModule_ReplyWithSimpleString(ctx, "OK"); + } else { + return RedisModule_ReplyWithError(ctx, "Global user currently not used"); + } +} + +/* HELLOACL.RESET + * Synchronously delete and re-create a module user. */ +int ResetCommand_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + RedisModule_FreeModuleUser(global); + global = RedisModule_CreateModuleUser("global"); + RedisModule_SetModuleUserACL(global, "allcommands"); + RedisModule_SetModuleUserACL(global, "allkeys"); + RedisModule_SetModuleUserACL(global, "on"); + + return RedisModule_ReplyWithSimpleString(ctx, "OK"); +} + +/* Callback handler for user changes, use this to notify a module of + * changes to users authenticated by the module */ +void HelloACL_UserChanged(uint64_t client_id, void *privdata) { + REDISMODULE_NOT_USED(privdata); + REDISMODULE_NOT_USED(client_id); + global_auth_client_id = 0; +} + +/* HELLOACL.AUTHGLOBAL + * Synchronously assigns a module user to the current context. */ +int AuthGlobalCommand_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (global_auth_client_id) { + return RedisModule_ReplyWithError(ctx, "Global user currently used"); + } + + RedisModule_AuthenticateClientWithUser(ctx, global, HelloACL_UserChanged, NULL, &global_auth_client_id); + + return RedisModule_ReplyWithSimpleString(ctx, "OK"); +} + +#define TIMEOUT_TIME 1000 + +/* Reply callback for auth command HELLOACL.AUTHASYNC */ +int HelloACL_Reply(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + size_t length; + + RedisModuleString *user_string = RedisModule_GetBlockedClientPrivateData(ctx); + const char *name = RedisModule_StringPtrLen(user_string, &length); + + if (RedisModule_AuthenticateClientWithACLUser(ctx, name, length, NULL, NULL, NULL) == + REDISMODULE_ERR) { + return RedisModule_ReplyWithError(ctx, "Invalid Username or password"); + } + return RedisModule_ReplyWithSimpleString(ctx, "OK"); +} + +/* Timeout callback for auth command HELLOACL.AUTHASYNC */ +int HelloACL_Timeout(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + return RedisModule_ReplyWithSimpleString(ctx, "Request timedout"); +} + +/* Private data frees data for HELLOACL.AUTHASYNC command. */ +void HelloACL_FreeData(RedisModuleCtx *ctx, void *privdata) { + REDISMODULE_NOT_USED(ctx); + RedisModule_FreeString(NULL, privdata); +} + +/* Background authentication can happen here. */ +void *HelloACL_ThreadMain(void *args) { + void **targs = args; + RedisModuleBlockedClient *bc = targs[0]; + RedisModuleString *user = targs[1]; + RedisModule_Free(targs); + + RedisModule_UnblockClient(bc,user); + return NULL; +} + +/* HELLOACL.AUTHASYNC + * Asynchronously assigns an ACL user to the current context. */ +int AuthAsyncCommand_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (argc != 2) return RedisModule_WrongArity(ctx); + + pthread_t tid; + RedisModuleBlockedClient *bc = RedisModule_BlockClient(ctx, HelloACL_Reply, HelloACL_Timeout, HelloACL_FreeData, TIMEOUT_TIME); + + + void **targs = RedisModule_Alloc(sizeof(void*)*2); + targs[0] = bc; + targs[1] = RedisModule_CreateStringFromString(NULL, argv[1]); + + if (pthread_create(&tid, NULL, HelloACL_ThreadMain, targs) != 0) { + RedisModule_AbortBlock(bc); + return RedisModule_ReplyWithError(ctx, "-ERR Can't start thread"); + } + + return REDISMODULE_OK; +} + +/* This function must be present on each Redis module. It is used in order to + * register the commands into the Redis server. */ +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (RedisModule_Init(ctx,"helloacl",1,REDISMODULE_APIVER_1) + == REDISMODULE_ERR) return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"helloacl.reset", + ResetCommand_RedisCommand,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"helloacl.revoke", + RevokeCommand_RedisCommand,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"helloacl.authglobal", + AuthGlobalCommand_RedisCommand,"no-auth",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"helloacl.authasync", + AuthAsyncCommand_RedisCommand,"no-auth",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + global = RedisModule_CreateModuleUser("global"); + RedisModule_SetModuleUserACL(global, "allcommands"); + RedisModule_SetModuleUserACL(global, "allkeys"); + RedisModule_SetModuleUserACL(global, "on"); + + global_auth_client_id = 0; + + return REDISMODULE_OK; +} diff --git a/redis.submodule/src/modules/hellohook.c b/redis.submodule/src/modules/hellohook.c new file mode 100644 index 0000000..7ab78ed --- /dev/null +++ b/redis.submodule/src/modules/hellohook.c @@ -0,0 +1,93 @@ +/* Server hooks API example + * + * ----------------------------------------------------------------------------- + * + * Copyright (c) 2019, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define REDISMODULE_EXPERIMENTAL_API +#include "../redismodule.h" +#include +#include +#include +#include + +/* Client state change callback. */ +void clientChangeCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(ctx); + REDISMODULE_NOT_USED(e); + + RedisModuleClientInfo *ci = data; + printf("Client %s event for client #%llu %s:%d\n", + (sub == REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED) ? + "connection" : "disconnection", + ci->id,ci->addr,ci->port); +} + +void flushdbCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(ctx); + REDISMODULE_NOT_USED(e); + + RedisModuleFlushInfo *fi = data; + if (sub == REDISMODULE_SUBEVENT_FLUSHDB_START) { + if (fi->dbnum != -1) { + RedisModuleCallReply *reply; + reply = RedisModule_Call(ctx,"DBSIZE",""); + long long numkeys = RedisModule_CallReplyInteger(reply); + printf("FLUSHDB event of database %d started (%lld keys in DB)\n", + fi->dbnum, numkeys); + RedisModule_FreeCallReply(reply); + } else { + printf("FLUSHALL event started\n"); + } + } else { + if (fi->dbnum != -1) { + printf("FLUSHDB event of database %d ended\n",fi->dbnum); + } else { + printf("FLUSHALL event ended\n"); + } + } +} + +/* This function must be present on each Redis module. It is used in order to + * register the commands into the Redis server. */ +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (RedisModule_Init(ctx,"hellohook",1,REDISMODULE_APIVER_1) + == REDISMODULE_ERR) return REDISMODULE_ERR; + + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_ClientChange, clientChangeCallback); + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_FlushDB, flushdbCallback); + return REDISMODULE_OK; +} diff --git a/redis.submodule/src/modules/hellotype.c b/redis.submodule/src/modules/hellotype.c index ba634c4..4f2d1d7 100644 --- a/redis.submodule/src/modules/hellotype.c +++ b/redis.submodule/src/modules/hellotype.c @@ -129,6 +129,7 @@ int HelloTypeInsert_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, /* Insert the new element. */ HelloTypeInsert(hto,value); + RedisModule_SignalKeyAsReady(ctx,argv[1]); RedisModule_ReplyWithLongLong(ctx,hto->len); RedisModule_ReplicateVerbatim(ctx); @@ -190,6 +191,77 @@ int HelloTypeLen_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int return REDISMODULE_OK; } +/* ====================== Example of a blocking command ==================== */ + +/* Reply callback for blocking command HELLOTYPE.BRANGE, this will get + * called when the key we blocked for is ready: we need to check if we + * can really serve the client, and reply OK or ERR accordingly. */ +int HelloBlock_Reply(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + RedisModuleString *keyname = RedisModule_GetBlockedClientReadyKey(ctx); + RedisModuleKey *key = RedisModule_OpenKey(ctx,keyname,REDISMODULE_READ); + int type = RedisModule_KeyType(key); + if (type != REDISMODULE_KEYTYPE_MODULE || + RedisModule_ModuleTypeGetType(key) != HelloType) + { + RedisModule_CloseKey(key); + return REDISMODULE_ERR; + } + + /* In case the key is able to serve our blocked client, let's directly + * use our original command implementation to make this example simpler. */ + RedisModule_CloseKey(key); + return HelloTypeRange_RedisCommand(ctx,argv,argc-1); +} + +/* Timeout callback for blocking command HELLOTYPE.BRANGE */ +int HelloBlock_Timeout(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + return RedisModule_ReplyWithSimpleString(ctx,"Request timedout"); +} + +/* Private data freeing callback for HELLOTYPE.BRANGE command. */ +void HelloBlock_FreeData(RedisModuleCtx *ctx, void *privdata) { + REDISMODULE_NOT_USED(ctx); + RedisModule_Free(privdata); +} + +/* HELLOTYPE.BRANGE key first count timeout -- This is a blocking verison of + * the RANGE operation, in order to show how to use the API + * RedisModule_BlockClientOnKeys(). */ +int HelloTypeBRange_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (argc != 5) return RedisModule_WrongArity(ctx); + RedisModule_AutoMemory(ctx); /* Use automatic memory management. */ + RedisModuleKey *key = RedisModule_OpenKey(ctx,argv[1], + REDISMODULE_READ|REDISMODULE_WRITE); + int type = RedisModule_KeyType(key); + if (type != REDISMODULE_KEYTYPE_EMPTY && + RedisModule_ModuleTypeGetType(key) != HelloType) + { + return RedisModule_ReplyWithError(ctx,REDISMODULE_ERRORMSG_WRONGTYPE); + } + + /* Parse the timeout before even trying to serve the client synchronously, + * so that we always fail ASAP on syntax errors. */ + long long timeout; + if (RedisModule_StringToLongLong(argv[4],&timeout) != REDISMODULE_OK) { + return RedisModule_ReplyWithError(ctx, + "ERR invalid timeout parameter"); + } + + /* Can we serve the reply synchronously? */ + if (type != REDISMODULE_KEYTYPE_EMPTY) { + return HelloTypeRange_RedisCommand(ctx,argv,argc-1); + } + + /* Otherwise let's block on the key. */ + void *privdata = RedisModule_Alloc(100); + RedisModule_BlockClientOnKeys(ctx,HelloBlock_Reply,HelloBlock_Timeout,HelloBlock_FreeData,timeout,argv+1,1,privdata); + return REDISMODULE_OK; +} /* ========================== "hellotype" type methods ======================= */ @@ -282,5 +354,9 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) HelloTypeLen_RedisCommand,"readonly",1,1,1) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"hellotype.brange", + HelloTypeBRange_RedisCommand,"readonly",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + return REDISMODULE_OK; } diff --git a/redis.submodule/src/modules/testmodule.c b/redis.submodule/src/modules/testmodule.c index 67a8617..5381380 100644 --- a/redis.submodule/src/modules/testmodule.c +++ b/redis.submodule/src/modules/testmodule.c @@ -109,9 +109,9 @@ int TestStringPrintf(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { if (argc < 3) { return RedisModule_WrongArity(ctx); } - RedisModuleString *s = RedisModule_CreateStringPrintf(ctx, - "Got %d args. argv[1]: %s, argv[2]: %s", - argc, + RedisModuleString *s = RedisModule_CreateStringPrintf(ctx, + "Got %d args. argv[1]: %s, argv[2]: %s", + argc, RedisModule_StringPtrLen(argv[1], NULL), RedisModule_StringPtrLen(argv[2], NULL) ); @@ -133,7 +133,7 @@ int TestUnlink(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { RedisModuleKey *k = RedisModule_OpenKey(ctx, RedisModule_CreateStringPrintf(ctx, "unlinked"), REDISMODULE_WRITE | REDISMODULE_READ); if (!k) return failTest(ctx, "Could not create key"); - + if (REDISMODULE_ERR == RedisModule_StringSet(k, RedisModule_CreateStringPrintf(ctx, "Foobar"))) { return failTest(ctx, "Could not set string value"); } @@ -152,7 +152,7 @@ int TestUnlink(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { return failTest(ctx, "Could not verify key to be unlinked"); } return RedisModule_ReplyWithSimpleString(ctx, "OK"); - + } int NotifyCallback(RedisModuleCtx *ctx, int type, const char *event, @@ -188,6 +188,10 @@ int TestNotifications(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { RedisModule_Call(ctx, "LPUSH", "cc", "l", "y"); RedisModule_Call(ctx, "LPUSH", "cc", "l", "y"); + /* Miss some keys intentionally so we will get a "keymiss" notification. */ + RedisModule_Call(ctx, "GET", "c", "nosuchkey"); + RedisModule_Call(ctx, "SMEMBERS", "c", "nosuchkey"); + size_t sz; const char *rep; RedisModuleCallReply *r = RedisModule_Call(ctx, "HGET", "cc", "notifications", "foo"); @@ -225,6 +229,16 @@ int TestNotifications(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { FAIL("Wrong reply for l"); } + r = RedisModule_Call(ctx, "HGET", "cc", "notifications", "nosuchkey"); + if (r == NULL || RedisModule_CallReplyType(r) != REDISMODULE_REPLY_STRING) { + FAIL("Wrong or no reply for nosuchkey"); + } else { + rep = RedisModule_CallReplyStringPtr(r, &sz); + if (sz != 1 || *rep != '2') { + FAIL("Got reply '%.*s'. expected '2'", sz, rep); + } + } + RedisModule_Call(ctx, "FLUSHDB", ""); return RedisModule_ReplyWithSimpleString(ctx, "OK"); @@ -423,7 +437,7 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) if (RedisModule_CreateCommand(ctx,"test.ctxflags", TestCtxFlags,"readonly",1,1,1) == REDISMODULE_ERR) return REDISMODULE_ERR; - + if (RedisModule_CreateCommand(ctx,"test.unlink", TestUnlink,"write deny-oom",1,1,1) == REDISMODULE_ERR) return REDISMODULE_ERR; @@ -435,7 +449,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) RedisModule_SubscribeToKeyspaceEvents(ctx, REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_SET | - REDISMODULE_NOTIFY_STRING, + REDISMODULE_NOTIFY_STRING | + REDISMODULE_NOTIFY_KEY_MISS, NotifyCallback); if (RedisModule_CreateCommand(ctx,"test.notify", TestNotifications,"write deny-oom",1,1,1) == REDISMODULE_ERR) diff --git a/redis.submodule/src/multi.c b/redis.submodule/src/multi.c index 5971f46..60a07df 100644 --- a/redis.submodule/src/multi.c +++ b/redis.submodule/src/multi.c @@ -58,6 +58,13 @@ void queueMultiCommand(client *c) { multiCmd *mc; int j; + /* No sense to waste memory if the transaction is already aborted. + * this is useful in case client sends these in a pipeline, or doesn't + * bother to read previous responses and didn't notice the multi was already + * aborted. */ + if (c->flags & CLIENT_DIRTY_EXEC) + return; + c->mstate.commands = zrealloc(c->mstate.commands, sizeof(multiCmd)*(c->mstate.count+1)); mc = c->mstate.commands+c->mstate.count; @@ -106,11 +113,13 @@ void discardCommand(client *c) { /* Send a MULTI command to all the slaves and AOF file. Check the execCommand * implementation for more information. */ void execCommandPropagateMulti(client *c) { - robj *multistring = createStringObject("MULTI",5); + propagate(server.multiCommand,c->db->id,&shared.multi,1, + PROPAGATE_AOF|PROPAGATE_REPL); +} - propagate(server.multiCommand,c->db->id,&multistring,1, +void execCommandPropagateExec(client *c) { + propagate(server.execCommand,c->db->id,&shared.exec,1, PROPAGATE_AOF|PROPAGATE_REPL); - decrRefCount(multistring); } void execCommand(client *c) { @@ -126,6 +135,15 @@ void execCommand(client *c) { return; } + /* If we are in -BUSY state, flag the transaction and return the + * -BUSY error, like Redis <= 5. This is a temporary fix, may be changed + * ASAP, see issue #7353 on Github. */ + if (server.lua_timedout) { + flagTransaction(c); + addReply(c, shared.slowscripterr); + return; + } + /* Check if we need to abort the EXEC because: * 1) Some WATCHed key was touched. * 2) There was a previous error while queueing commands. @@ -134,7 +152,7 @@ void execCommand(client *c) { * in the second an EXECABORT error is returned. */ if (c->flags & (CLIENT_DIRTY_CAS|CLIENT_DIRTY_EXEC)) { addReply(c, c->flags & CLIENT_DIRTY_EXEC ? shared.execaborterr : - shared.nullmultibulk); + shared.nullarray[c->resp]); discardTransaction(c); goto handle_monitor; } @@ -149,7 +167,7 @@ void execCommand(client *c) { { addReplyError(c, "Transaction contains write commands but instance " - "is now a read-only slave. EXEC aborted."); + "is now a read-only replica. EXEC aborted."); discardTransaction(c); goto handle_monitor; } @@ -159,7 +177,7 @@ void execCommand(client *c) { orig_argv = c->argv; orig_argc = c->argc; orig_cmd = c->cmd; - addReplyMultiBulkLen(c,c->mstate.count); + addReplyArrayLen(c,c->mstate.count); for (j = 0; j < c->mstate.count; j++) { c->argc = c->mstate.commands[j].argc; c->argv = c->mstate.commands[j].argv; @@ -170,12 +188,29 @@ void execCommand(client *c) { * This way we'll deliver the MULTI/..../EXEC block as a whole and * both the AOF and the replication link will have the same consistency * and atomicity guarantees. */ - if (!must_propagate && !(c->cmd->flags & (CMD_READONLY|CMD_ADMIN))) { + if (!must_propagate && + !server.loading && + !(c->cmd->flags & (CMD_READONLY|CMD_ADMIN))) + { execCommandPropagateMulti(c); must_propagate = 1; } - call(c,server.loading ? CMD_CALL_NONE : CMD_CALL_FULL); + int acl_keypos; + int acl_retval = ACLCheckCommandPerm(c,&acl_keypos); + if (acl_retval != ACL_OK) { + addACLLogEntry(c,acl_retval,acl_keypos,NULL); + addReplyErrorFormat(c, + "-NOPERM ACLs rules changed between the moment the " + "transaction was accumulated and the EXEC call. " + "This command is no longer allowed for the " + "following reason: %s", + (acl_retval == ACL_DENIED_CMD) ? + "no permission to execute the command or subcommand" : + "no permission to touch the specified keys"); + } else { + call(c,server.loading ? CMD_CALL_NONE : CMD_CALL_FULL); + } /* Commands may alter argc/argv, restore mstate. */ c->mstate.commands[j].argc = c->argc; diff --git a/redis.submodule/src/networking.c b/redis.submodule/src/networking.c index e2af03d..77b9a6f 100644 --- a/redis.submodule/src/networking.c +++ b/redis.submodule/src/networking.c @@ -35,6 +35,8 @@ #include static void setProtocolError(const char *errstr, client *c); +int postponeClientRead(client *c); +int ProcessingEventsWhileBlocked = 0; /* See processEventsWhileBlocked(). */ /* Return the size consumed from the allocator, for the specified SDS string, * including internal fragmentation. This function is used in order to compute @@ -83,32 +85,27 @@ void linkClient(client *c) { raxInsert(server.clients_index,(unsigned char*)&id,sizeof(id),c,NULL); } -client *createClient(int fd) { +client *createClient(connection *conn) { client *c = zmalloc(sizeof(client)); - /* passing -1 as fd it is possible to create a non connected client. + /* passing NULL as conn it is possible to create a non connected client. * This is useful since all the commands needs to be executed * in the context of a client. When commands are executed in other * contexts (for instance a Lua script) we need a non connected client. */ - if (fd != -1) { - anetNonBlock(NULL,fd); - anetEnableTcpNoDelay(NULL,fd); + if (conn) { + connNonBlock(conn); + connEnableTcpNoDelay(conn); if (server.tcpkeepalive) - anetKeepAlive(NULL,fd,server.tcpkeepalive); - if (aeCreateFileEvent(server.el,fd,AE_READABLE, - readQueryFromClient, c) == AE_ERR) - { - close(fd); - zfree(c); - return NULL; - } + connKeepAlive(conn,server.tcpkeepalive); + connSetReadHandler(conn, readQueryFromClient); + connSetPrivateData(conn, c); } selectDb(c,0); - uint64_t client_id; - atomicGetIncr(server.next_client_id,client_id,1); + uint64_t client_id = ++server.next_client_id; c->id = client_id; - c->fd = fd; + c->resp = 2; + c->conn = conn; c->name = NULL; c->bufpos = 0; c->qb_pos = 0; @@ -119,12 +116,16 @@ client *createClient(int fd) { c->argc = 0; c->argv = NULL; c->cmd = c->lastcmd = NULL; + c->user = DefaultUser; c->multibulklen = 0; c->bulklen = -1; c->sentlen = 0; c->flags = 0; c->ctime = c->lastinteraction = server.unixtime; - c->authenticated = 0; + /* If the default user does not require authentication, the user is + * directly authenticated. */ + c->authenticated = (c->user->flags & USER_FLAG_NOPASS) && + !(c->user->flags & USER_FLAG_DISABLED); c->replstate = REPL_STATE_NONE; c->repl_put_online_on_ack = 0; c->reploff = 0; @@ -154,9 +155,16 @@ client *createClient(int fd) { c->pubsub_patterns = listCreate(); c->peerid = NULL; c->client_list_node = NULL; + c->client_tracking_redirection = 0; + c->client_tracking_prefixes = NULL; + c->client_cron_last_memory_usage = 0; + c->client_cron_last_memory_type = CLIENT_TYPE_NORMAL; + c->auth_callback = NULL; + c->auth_callback_privdata = NULL; + c->auth_module = NULL; listSetFreeMethod(c->pubsub_patterns,decrRefCountVoid); listSetMatchMethod(c->pubsub_patterns,listMatchObjects); - if (fd != -1) linkClient(c); + if (conn) linkClient(c); initClientMultiState(c); return c; } @@ -222,7 +230,7 @@ int prepareClientToWrite(client *c) { if ((c->flags & CLIENT_MASTER) && !(c->flags & CLIENT_MASTER_FORCE_REPLY)) return C_ERR; - if (c->fd <= 0) return C_ERR; /* Fake client for AOF loading. */ + if (!c->conn) return C_ERR; /* Fake client for AOF loading. */ /* Schedule the client to write the output buffers to the socket, unless * it should already be setup to do so (it has already pending data). */ @@ -253,14 +261,14 @@ int _addReplyToBuffer(client *c, const char *s, size_t len) { return C_OK; } -void _addReplyStringToList(client *c, const char *s, size_t len) { +void _addReplyProtoToList(client *c, const char *s, size_t len) { if (c->flags & CLIENT_CLOSE_AFTER_REPLY) return; listNode *ln = listLast(c->reply); clientReplyBlock *tail = ln? listNodeValue(ln): NULL; /* Note that 'tail' may be NULL even if we have a tail node, becuase when - * addDeferredMultiBulkLength() is used, it sets a dummy node to NULL just + * addReplyDeferredLen() is used, it sets a dummy node to NULL just * fo fill it later, when the size of the bulk length is set. */ /* Append to tail string when possible. */ @@ -300,7 +308,7 @@ void addReply(client *c, robj *obj) { if (sdsEncodedObject(obj)) { if (_addReplyToBuffer(c,obj->ptr,sdslen(obj->ptr)) != C_OK) - _addReplyStringToList(c,obj->ptr,sdslen(obj->ptr)); + _addReplyProtoToList(c,obj->ptr,sdslen(obj->ptr)); } else if (obj->encoding == OBJ_ENCODING_INT) { /* For integer encoded strings we just convert it into a string * using our optimized function, and attach the resulting string @@ -308,7 +316,7 @@ void addReply(client *c, robj *obj) { char buf[32]; size_t len = ll2string(buf,sizeof(buf),(long)obj->ptr); if (_addReplyToBuffer(c,buf,len) != C_OK) - _addReplyStringToList(c,buf,len); + _addReplyProtoToList(c,buf,len); } else { serverPanic("Wrong obj->encoding in addReply()"); } @@ -323,7 +331,7 @@ void addReplySds(client *c, sds s) { return; } if (_addReplyToBuffer(c,s,sdslen(s)) != C_OK) - _addReplyStringToList(c,s,sdslen(s)); + _addReplyProtoToList(c,s,sdslen(s)); sdsfree(s); } @@ -333,12 +341,12 @@ void addReplySds(client *c, sds s) { * * It is efficient because does not create an SDS object nor an Redis object * if not needed. The object will only be created by calling - * _addReplyStringToList() if we fail to extend the existing tail object + * _addReplyProtoToList() if we fail to extend the existing tail object * in the list of objects. */ -void addReplyString(client *c, const char *s, size_t len) { +void addReplyProto(client *c, const char *s, size_t len) { if (prepareClientToWrite(c) != C_OK) return; if (_addReplyToBuffer(c,s,len) != C_OK) - _addReplyStringToList(c,s,len); + _addReplyProtoToList(c,s,len); } /* Low level function called by the addReplyError...() functions. @@ -352,9 +360,9 @@ void addReplyString(client *c, const char *s, size_t len) { void addReplyErrorLength(client *c, const char *s, size_t len) { /* If the string already starts with "-..." then the error code * is provided by the caller. Otherwise we use "-ERR". */ - if (!len || s[0] != '-') addReplyString(c,"-ERR ",5); - addReplyString(c,s,len); - addReplyString(c,"\r\n",2); + if (!len || s[0] != '-') addReplyProto(c,"-ERR ",5); + addReplyProto(c,s,len); + addReplyProto(c,"\r\n",2); /* Sometimes it could be normal that a slave replies to a master with * an error and this function gets called. Actually the error will never @@ -366,13 +374,31 @@ void addReplyErrorLength(client *c, const char *s, size_t len) { * Where the master must propagate the first change even if the second * will produce an error. However it is useful to log such events since * they are rare and may hint at errors in a script or a bug in Redis. */ - if (c->flags & (CLIENT_MASTER|CLIENT_SLAVE) && !(c->flags & CLIENT_MONITOR)) { - char* to = c->flags & CLIENT_MASTER? "master": "replica"; - char* from = c->flags & CLIENT_MASTER? "replica": "master"; + int ctype = getClientType(c); + if (ctype == CLIENT_TYPE_MASTER || ctype == CLIENT_TYPE_SLAVE || c->id == CLIENT_ID_AOF) { + char *to, *from; + + if (c->id == CLIENT_ID_AOF) { + to = "AOF-loading-client"; + from = "server"; + } else if (ctype == CLIENT_TYPE_MASTER) { + to = "master"; + from = "replica"; + } else { + to = "replica"; + from = "master"; + } + char *cmdname = c->lastcmd ? c->lastcmd->name : ""; serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error " "to its %s: '%s' after processing the command " "'%s'", from, to, s, cmdname); + if (ctype == CLIENT_TYPE_MASTER && server.repl_backlog && + server.repl_backlog_histlen > 0) + { + showLatestBacklog(); + } + server.stat_unexpected_error_replies++; } } @@ -397,9 +423,9 @@ void addReplyErrorFormat(client *c, const char *fmt, ...) { } void addReplyStatusLength(client *c, const char *s, size_t len) { - addReplyString(c,"+",1); - addReplyString(c,s,len); - addReplyString(c,"\r\n",2); + addReplyProto(c,"+",1); + addReplyProto(c,s,len); + addReplyProto(c,"\r\n",2); } void addReplyStatus(client *c, const char *status) { @@ -415,30 +441,59 @@ void addReplyStatusFormat(client *c, const char *fmt, ...) { sdsfree(s); } +/* Sometimes we are forced to create a new reply node, and we can't append to + * the previous one, when that happens, we wanna try to trim the unused space + * at the end of the last reply node which we won't use anymore. */ +void trimReplyUnusedTailSpace(client *c) { + listNode *ln = listLast(c->reply); + clientReplyBlock *tail = ln? listNodeValue(ln): NULL; + + /* Note that 'tail' may be NULL even if we have a tail node, becuase when + * addReplyDeferredLen() is used */ + if (!tail) return; + + /* We only try to trim the space is relatively high (more than a 1/4 of the + * allocation), otherwise there's a high chance realloc will NOP. + * Also, to avoid large memmove which happens as part of realloc, we only do + * that if the used part is small. */ + if (tail->size - tail->used > tail->size / 4 && + tail->used < PROTO_REPLY_CHUNK_BYTES) + { + size_t old_size = tail->size; + tail = zrealloc(tail, tail->used + sizeof(clientReplyBlock)); + /* take over the allocation's internal fragmentation (at least for + * memory usage tracking) */ + tail->size = zmalloc_usable(tail) - sizeof(clientReplyBlock); + c->reply_bytes = c->reply_bytes + tail->size - old_size; + listNodeValue(ln) = tail; + } +} + /* Adds an empty object to the reply list that will contain the multi bulk * length, which is not known when this function is called. */ -void *addDeferredMultiBulkLength(client *c) { +void *addReplyDeferredLen(client *c) { /* Note that we install the write event here even if the object is not * ready to be sent, since we are sure that before returning to the - * event loop setDeferredMultiBulkLength() will be called. */ + * event loop setDeferredAggregateLen() will be called. */ if (prepareClientToWrite(c) != C_OK) return NULL; + trimReplyUnusedTailSpace(c); listAddNodeTail(c->reply,NULL); /* NULL is our placeholder. */ return listLast(c->reply); } /* Populate the length object and try gluing it to the next chunk. */ -void setDeferredMultiBulkLength(client *c, void *node, long length) { +void setDeferredAggregateLen(client *c, void *node, long length, char prefix) { listNode *ln = (listNode*)node; clientReplyBlock *next; char lenstr[128]; - size_t lenstr_len = sprintf(lenstr, "*%ld\r\n", length); + size_t lenstr_len = sprintf(lenstr, "%c%ld\r\n", prefix, length); /* Abort when *node is NULL: when the client should not accept writes - * we return NULL in addDeferredMultiBulkLength() */ + * we return NULL in addReplyDeferredLen() */ if (node == NULL) return; serverAssert(!listNodeValue(ln)); - /* Normally we fill this dummy NULL node, added by addDeferredMultiBulkLength(), + /* Normally we fill this dummy NULL node, added by addReplyDeferredLen(), * with a new buffer structure containing the protocol needed to specify * the length of the array following. However sometimes when there is * little memory to move, we may instead remove this NULL node, and prefix @@ -468,18 +523,55 @@ void setDeferredMultiBulkLength(client *c, void *node, long length) { asyncCloseClientOnOutputBufferLimitReached(c); } +void setDeferredArrayLen(client *c, void *node, long length) { + setDeferredAggregateLen(c,node,length,'*'); +} + +void setDeferredMapLen(client *c, void *node, long length) { + int prefix = c->resp == 2 ? '*' : '%'; + if (c->resp == 2) length *= 2; + setDeferredAggregateLen(c,node,length,prefix); +} + +void setDeferredSetLen(client *c, void *node, long length) { + int prefix = c->resp == 2 ? '*' : '~'; + setDeferredAggregateLen(c,node,length,prefix); +} + +void setDeferredAttributeLen(client *c, void *node, long length) { + int prefix = c->resp == 2 ? '*' : '|'; + if (c->resp == 2) length *= 2; + setDeferredAggregateLen(c,node,length,prefix); +} + +void setDeferredPushLen(client *c, void *node, long length) { + int prefix = c->resp == 2 ? '*' : '>'; + setDeferredAggregateLen(c,node,length,prefix); +} + /* Add a double as a bulk reply */ void addReplyDouble(client *c, double d) { - char dbuf[128], sbuf[128]; - int dlen, slen; if (isinf(d)) { /* Libc in odd systems (Hi Solaris!) will format infinite in a * different way, so better to handle it in an explicit way. */ - addReplyBulkCString(c, d > 0 ? "inf" : "-inf"); + if (c->resp == 2) { + addReplyBulkCString(c, d > 0 ? "inf" : "-inf"); + } else { + addReplyProto(c, d > 0 ? ",inf\r\n" : ",-inf\r\n", + d > 0 ? 6 : 7); + } } else { - dlen = snprintf(dbuf,sizeof(dbuf),"%.17g",d); - slen = snprintf(sbuf,sizeof(sbuf),"$%d\r\n%s\r\n",dlen,dbuf); - addReplyString(c,sbuf,slen); + char dbuf[MAX_LONG_DOUBLE_CHARS+3], + sbuf[MAX_LONG_DOUBLE_CHARS+32]; + int dlen, slen; + if (c->resp == 2) { + dlen = snprintf(dbuf,sizeof(dbuf),"%.17g",d); + slen = snprintf(sbuf,sizeof(sbuf),"$%d\r\n%s\r\n",dlen,dbuf); + addReplyProto(c,sbuf,slen); + } else { + dlen = snprintf(dbuf,sizeof(dbuf),",%.17g\r\n",d); + addReplyProto(c,dbuf,dlen); + } } } @@ -487,9 +579,17 @@ void addReplyDouble(client *c, double d) { * of the double instead of exposing the crude behavior of doubles to the * dear user. */ void addReplyHumanLongDouble(client *c, long double d) { - robj *o = createStringObjectFromLongDouble(d,1); - addReplyBulk(c,o); - decrRefCount(o); + if (c->resp == 2) { + robj *o = createStringObjectFromLongDouble(d,1); + addReplyBulk(c,o); + decrRefCount(o); + } else { + char buf[MAX_LONG_DOUBLE_CHARS]; + int len = ld2string(buf,sizeof(buf),d,LD_STR_HUMAN); + addReplyProto(c,",",1); + addReplyProto(c,buf,len); + addReplyProto(c,"\r\n",2); + } } /* Add a long long as integer reply or bulk len / multi bulk count. @@ -513,7 +613,7 @@ void addReplyLongLongWithPrefix(client *c, long long ll, char prefix) { len = ll2string(buf+1,sizeof(buf)-1,ll); buf[len+1] = '\r'; buf[len+2] = '\n'; - addReplyString(c,buf,len+3); + addReplyProto(c,buf,len+3); } void addReplyLongLong(client *c, long long ll) { @@ -525,32 +625,70 @@ void addReplyLongLong(client *c, long long ll) { addReplyLongLongWithPrefix(c,ll,':'); } -void addReplyMultiBulkLen(client *c, long length) { - if (length < OBJ_SHARED_BULKHDR_LEN) +void addReplyAggregateLen(client *c, long length, int prefix) { + if (prefix == '*' && length < OBJ_SHARED_BULKHDR_LEN) addReply(c,shared.mbulkhdr[length]); else - addReplyLongLongWithPrefix(c,length,'*'); + addReplyLongLongWithPrefix(c,length,prefix); } -/* Create the length prefix of a bulk reply, example: $2234 */ -void addReplyBulkLen(client *c, robj *obj) { - size_t len; +void addReplyArrayLen(client *c, long length) { + addReplyAggregateLen(c,length,'*'); +} - if (sdsEncodedObject(obj)) { - len = sdslen(obj->ptr); +void addReplyMapLen(client *c, long length) { + int prefix = c->resp == 2 ? '*' : '%'; + if (c->resp == 2) length *= 2; + addReplyAggregateLen(c,length,prefix); +} + +void addReplySetLen(client *c, long length) { + int prefix = c->resp == 2 ? '*' : '~'; + addReplyAggregateLen(c,length,prefix); +} + +void addReplyAttributeLen(client *c, long length) { + int prefix = c->resp == 2 ? '*' : '|'; + if (c->resp == 2) length *= 2; + addReplyAggregateLen(c,length,prefix); +} + +void addReplyPushLen(client *c, long length) { + int prefix = c->resp == 2 ? '*' : '>'; + addReplyAggregateLen(c,length,prefix); +} + +void addReplyNull(client *c) { + if (c->resp == 2) { + addReplyProto(c,"$-1\r\n",5); } else { - long n = (long)obj->ptr; + addReplyProto(c,"_\r\n",3); + } +} - /* Compute how many bytes will take this integer as a radix 10 string */ - len = 1; - if (n < 0) { - len++; - n = -n; - } - while((n = n/10) != 0) { - len++; - } +void addReplyBool(client *c, int b) { + if (c->resp == 2) { + addReply(c, b ? shared.cone : shared.czero); + } else { + addReplyProto(c, b ? "#t\r\n" : "#f\r\n",4); + } +} + +/* A null array is a concept that no longer exists in RESP3. However + * RESP2 had it, so API-wise we have this call, that will emit the correct + * RESP2 protocol, however for RESP3 the reply will always be just the + * Null type "_\r\n". */ +void addReplyNullArray(client *c) { + if (c->resp == 2) { + addReplyProto(c,"*-1\r\n",5); + } else { + addReplyProto(c,"_\r\n",3); } +} + +/* Create the length prefix of a bulk reply, example: $2234 */ +void addReplyBulkLen(client *c, robj *obj) { + size_t len = stringObjectLen(obj); if (len < OBJ_SHARED_BULKHDR_LEN) addReply(c,shared.bulkhdr[len]); @@ -568,7 +706,7 @@ void addReplyBulk(client *c, robj *obj) { /* Add a C buffer as bulk reply */ void addReplyBulkCBuffer(client *c, const void *p, size_t len) { addReplyLongLongWithPrefix(c,len,'$'); - addReplyString(c,p,len); + addReplyProto(c,p,len); addReply(c,shared.crlf); } @@ -582,7 +720,7 @@ void addReplyBulkSds(client *c, sds s) { /* Add a C null term string as bulk reply */ void addReplyBulkCString(client *c, const char *s) { if (s == NULL) { - addReply(c,shared.nullbulk); + addReplyNull(c); } else { addReplyBulkCBuffer(c,s,strlen(s)); } @@ -597,13 +735,42 @@ void addReplyBulkLongLong(client *c, long long ll) { addReplyBulkCBuffer(c,buf,len); } +/* Reply with a verbatim type having the specified extension. + * + * The 'ext' is the "extension" of the file, actually just a three + * character type that describes the format of the verbatim string. + * For instance "txt" means it should be interpreted as a text only + * file by the receiver, "md " as markdown, and so forth. Only the + * three first characters of the extension are used, and if the + * provided one is shorter than that, the remaining is filled with + * spaces. */ +void addReplyVerbatim(client *c, const char *s, size_t len, const char *ext) { + if (c->resp == 2) { + addReplyBulkCBuffer(c,s,len); + } else { + char buf[32]; + size_t preflen = snprintf(buf,sizeof(buf),"=%zu\r\nxxx:",len+4); + char *p = buf+preflen-4; + for (int i = 0; i < 3; i++) { + if (*ext == '\0') { + p[i] = ' '; + } else { + p[i] = *ext++; + } + } + addReplyProto(c,buf,preflen); + addReplyProto(c,s,len); + addReplyProto(c,"\r\n",2); + } +} + /* Add an array of C strings as status replies with a heading. * This function is typically invoked by from commands that support * subcommands in response to the 'help' subcommand. The help array * is terminated by NULL sentinel. */ void addReplyHelp(client *c, const char **help) { sds cmd = sdsnew((char*) c->argv[0]->ptr); - void *blenp = addDeferredMultiBulkLength(c); + void *blenp = addReplyDeferredLen(c); int blen = 0; sdstoupper(cmd); @@ -614,7 +781,7 @@ void addReplyHelp(client *c, const char **help) { while (help[blen]) addReplyStatus(c,help[blen++]); blen++; /* Account for the header line(s). */ - setDeferredMultiBulkLength(c,blenp,blen); + setDeferredArrayLen(c,blenp,blen); } /* Add a suggestive error reply. @@ -634,7 +801,7 @@ void addReplySubcommandSyntaxError(client *c) { void AddReplyFromClient(client *dst, client *src) { if (prepareClientToWrite(dst) != C_OK) return; - addReplyString(dst,src->buf, src->bufpos); + addReplyProto(dst,src->buf, src->bufpos); if (listLength(src->reply)) listJoin(dst->reply,src->reply); dst->reply_bytes += src->reply_bytes; @@ -660,29 +827,14 @@ int clientHasPendingReplies(client *c) { return c->bufpos || listLength(c->reply); } -#define MAX_ACCEPTS_PER_CALL 1000 -static void acceptCommonHandler(int fd, int flags, char *ip) { - client *c; - if ((c = createClient(fd)) == NULL) { - serverLog(LL_WARNING, - "Error registering fd event for the new client: %s (fd=%d)", - strerror(errno),fd); - close(fd); /* May be already closed, just ignore errors */ - return; - } - /* If maxclient directive is set and this is one client more... close the - * connection. Note that we create the client instead to check before - * for this condition, since now the socket is already set in non-blocking - * mode and we can send an error for free using the Kernel I/O */ - if (listLength(server.clients) > server.maxclients) { - char *err = "-ERR max number of clients reached\r\n"; +void clientAcceptHandler(connection *conn) { + client *c = connGetPrivateData(conn); - /* That's a best effort error message, don't check write errors */ - if (write(c->fd,err,strlen(err)) == -1) { - /* Nothing to do, Just to avoid the warning... */ - } - server.stat_rejected_conn++; - freeClient(c); + if (connGetState(conn) != CONN_STATE_CONNECTED) { + serverLog(LL_WARNING, + "Error accepting a client connection: %s", + connGetLastError(conn)); + freeClientAsync(c); return; } @@ -692,11 +844,13 @@ static void acceptCommonHandler(int fd, int flags, char *ip) { * user what to do to fix it if needed. */ if (server.protected_mode && server.bindaddr_count == 0 && - server.requirepass == NULL && - !(flags & CLIENT_UNIX_SOCKET) && - ip != NULL) + DefaultUser->flags & USER_FLAG_NOPASS && + !(c->flags & CLIENT_UNIX_SOCKET)) { - if (strcmp(ip,"127.0.0.1") && strcmp(ip,"::1")) { + char cip[NET_IP_STR_LEN+1] = { 0 }; + connPeerToString(conn, cip, sizeof(cip)-1, NULL); + + if (strcmp(cip,"127.0.0.1") && strcmp(cip,"::1")) { char *err = "-DENIED Redis is running in protected mode because protected " "mode is enabled, no bind address was specified, no " @@ -718,17 +872,76 @@ static void acceptCommonHandler(int fd, int flags, char *ip) { "4) Setup a bind address or an authentication password. " "NOTE: You only need to do one of the above things in order for " "the server to start accepting connections from the outside.\r\n"; - if (write(c->fd,err,strlen(err)) == -1) { + if (connWrite(c->conn,err,strlen(err)) == -1) { /* Nothing to do, Just to avoid the warning... */ } server.stat_rejected_conn++; - freeClient(c); + freeClientAsync(c); return; } } server.stat_numconnections++; + moduleFireServerEvent(REDISMODULE_EVENT_CLIENT_CHANGE, + REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED, + c); +} + +#define MAX_ACCEPTS_PER_CALL 1000 +static void acceptCommonHandler(connection *conn, int flags, char *ip) { + client *c; + UNUSED(ip); + + /* Admission control will happen before a client is created and connAccept() + * called, because we don't want to even start transport-level negotiation + * if rejected. + */ + if (listLength(server.clients) >= server.maxclients) { + char *err = "-ERR max number of clients reached\r\n"; + + /* That's a best effort error message, don't check write errors. + * Note that for TLS connections, no handshake was done yet so nothing is written + * and the connection will just drop. + */ + if (connWrite(conn,err,strlen(err)) == -1) { + /* Nothing to do, Just to avoid the warning... */ + } + server.stat_rejected_conn++; + connClose(conn); + return; + } + + /* Create connection and client */ + if ((c = createClient(conn)) == NULL) { + char conninfo[100]; + serverLog(LL_WARNING, + "Error registering fd event for the new client: %s (conn: %s)", + connGetLastError(conn), + connGetInfo(conn, conninfo, sizeof(conninfo))); + connClose(conn); /* May be already closed, just ignore errors */ + return; + } + + /* Last chance to keep flags */ c->flags |= flags; + + /* Initiate accept. + * + * Note that connAccept() is free to do two things here: + * 1. Call clientAcceptHandler() immediately; + * 2. Schedule a future call to clientAcceptHandler(). + * + * Because of that, we must do nothing else afterwards. + */ + if (connAccept(conn, clientAcceptHandler) == C_ERR) { + char conninfo[100]; + if (connGetState(conn) == CONN_STATE_ERROR) + serverLog(LL_WARNING, + "Error accepting a client connection: %s (conn: %s)", + connGetLastError(conn), connGetInfo(conn, conninfo, sizeof(conninfo))); + freeClient(connGetPrivateData(conn)); + return; + } } void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) { @@ -747,7 +960,27 @@ void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) { return; } serverLog(LL_VERBOSE,"Accepted %s:%d", cip, cport); - acceptCommonHandler(cfd,0,cip); + acceptCommonHandler(connCreateAcceptedSocket(cfd),0,cip); + } +} + +void acceptTLSHandler(aeEventLoop *el, int fd, void *privdata, int mask) { + int cport, cfd, max = MAX_ACCEPTS_PER_CALL; + char cip[NET_IP_STR_LEN]; + UNUSED(el); + UNUSED(mask); + UNUSED(privdata); + + while(max--) { + cfd = anetTcpAccept(server.neterr, fd, cip, sizeof(cip), &cport); + if (cfd == ANET_ERR) { + if (errno != EWOULDBLOCK) + serverLog(LL_WARNING, + "Accepting client connection: %s", server.neterr); + return; + } + serverLog(LL_VERBOSE,"Accepted %s:%d", cip, cport); + acceptCommonHandler(connCreateAcceptedTLS(cfd, server.tls_auth_clients),0,cip); } } @@ -766,7 +999,7 @@ void acceptUnixHandler(aeEventLoop *el, int fd, void *privdata, int mask) { return; } serverLog(LL_VERBOSE,"Accepted connection to %s", server.unixsocket); - acceptCommonHandler(cfd,CLIENT_UNIX_SOCKET,NULL); + acceptCommonHandler(connCreateAcceptedSocket(cfd),CLIENT_UNIX_SOCKET,NULL); } } @@ -782,8 +1015,10 @@ static void freeClientArgv(client *c) { * when we resync with our own master and want to force all our slaves to * resync with us as well. */ void disconnectSlaves(void) { - while (listLength(server.slaves)) { - listNode *ln = listFirst(server.slaves); + listIter li; + listNode *ln; + listRewind(server.slaves,&li); + while((ln = listNext(&li))) { freeClient((client*)ln->value); } } @@ -797,10 +1032,10 @@ void unlinkClient(client *c) { /* If this is marked as current client unset it. */ if (server.current_client == c) server.current_client = NULL; - /* Certain operations must be done only if the client has an active socket. + /* Certain operations must be done only if the client has an active connection. * If the client was already unlinked or if it's a "fake client" the - * fd is already set to -1. */ - if (c->fd != -1) { + * conn is already set to NULL. */ + if (c->conn) { /* Remove from the list of active clients. */ if (c->client_list_node) { uint64_t id = htonu64(c->id); @@ -809,21 +1044,23 @@ void unlinkClient(client *c) { c->client_list_node = NULL; } - /* In the case of diskless replication the fork is writing to the - * sockets and just closing the fd isn't enough, if we don't also - * shutdown the socket the fork will continue to write to the slave - * and the salve will only find out that it was disconnected when - * it will finish reading the rdb. */ - if ((c->flags & CLIENT_SLAVE) && - (c->replstate == SLAVE_STATE_WAIT_BGSAVE_END)) { - shutdown(c->fd, SHUT_RDWR); + /* Check if this is a replica waiting for diskless replication (rdb pipe), + * in which case it needs to be cleaned from that list */ + if (c->flags & CLIENT_SLAVE && + c->replstate == SLAVE_STATE_WAIT_BGSAVE_END && + server.rdb_pipe_conns) + { + int i; + for (i=0; i < server.rdb_pipe_numconns; i++) { + if (server.rdb_pipe_conns[i] == c->conn) { + rdbPipeWriteHandlerConnRemoved(c->conn); + server.rdb_pipe_conns[i] = NULL; + break; + } + } } - - /* Unregister async I/O handlers and close the socket. */ - aeDeleteFileEvent(server.el,c->fd,AE_READABLE); - aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); - close(c->fd); - c->fd = -1; + connClose(c->conn); + c->conn = NULL; } /* Remove from the list of pending writes if needed. */ @@ -834,6 +1071,14 @@ void unlinkClient(client *c) { c->flags &= ~CLIENT_PENDING_WRITE; } + /* Remove from the list of pending reads if needed. */ + if (c->flags & CLIENT_PENDING_READ) { + ln = listSearchKey(server.clients_pending_read,c); + serverAssert(ln != NULL); + listDelNode(server.clients_pending_read,ln); + c->flags &= ~CLIENT_PENDING_READ; + } + /* When client was just unblocked because of a blocking operation, * remove it from the list of unblocked clients. */ if (c->flags & CLIENT_UNBLOCKED) { @@ -842,6 +1087,9 @@ void unlinkClient(client *c) { listDelNode(server.unblocked_clients,ln); c->flags &= ~CLIENT_UNBLOCKED; } + + /* Clear the tracking status. */ + if (c->flags & CLIENT_TRACKING) disableTracking(c); } void freeClient(client *c) { @@ -854,6 +1102,26 @@ void freeClient(client *c) { return; } + /* For connected clients, call the disconnection event of modules hooks. */ + if (c->conn) { + moduleFireServerEvent(REDISMODULE_EVENT_CLIENT_CHANGE, + REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED, + c); + } + + /* Notify module system that this client auth status changed. */ + moduleNotifyUserChanged(c); + + /* If this client was scheduled for async freeing we need to remove it + * from the queue. Note that we need to do this here, because later + * we may call replicationCacheMaster() and the client should already + * be removed from the list of clients to free. */ + if (c->flags & CLIENT_CLOSE_ASAP) { + ln = listSearchKey(server.clients_to_close,c); + serverAssert(ln != NULL); + listDelNode(server.clients_to_close,ln); + } + /* If it is our master that's beging disconnected we should make sure * to cache the state to try a partial resynchronization later. * @@ -861,17 +1129,15 @@ void freeClient(client *c) { * some unexpected state, by checking its flags. */ if (server.master && c->flags & CLIENT_MASTER) { serverLog(LL_WARNING,"Connection with master lost."); - if (!(c->flags & (CLIENT_CLOSE_AFTER_REPLY| - CLIENT_CLOSE_ASAP| - CLIENT_BLOCKED))) - { + if (!(c->flags & (CLIENT_PROTOCOL_ERROR|CLIENT_BLOCKED))) { + c->flags &= ~(CLIENT_CLOSE_ASAP|CLIENT_CLOSE_AFTER_REPLY); replicationCacheMaster(c); return; } } /* Log link disconnection with slave */ - if ((c->flags & CLIENT_SLAVE) && !(c->flags & CLIENT_MONITOR)) { + if (getClientType(c) == CLIENT_TYPE_SLAVE) { serverLog(LL_WARNING,"Connection with replica %s lost.", replicationGetSlaveName(c)); } @@ -918,22 +1184,24 @@ void freeClient(client *c) { /* We need to remember the time when we started to have zero * attached slaves, as after some time we'll free the replication * backlog. */ - if (c->flags & CLIENT_SLAVE && listLength(server.slaves) == 0) + if (getClientType(c) == CLIENT_TYPE_SLAVE && listLength(server.slaves) == 0) server.repl_no_slaves_since = server.unixtime; refreshGoodSlavesCount(); + /* Fire the replica change modules event. */ + if (c->replstate == SLAVE_STATE_ONLINE) + moduleFireServerEvent(REDISMODULE_EVENT_REPLICA_CHANGE, + REDISMODULE_SUBEVENT_REPLICA_CHANGE_OFFLINE, + NULL); } /* Master/slave cleanup Case 2: * we lost the connection with the master. */ if (c->flags & CLIENT_MASTER) replicationHandleMasterDisconnection(); - /* If this client was scheduled for async freeing we need to remove it - * from the queue. */ - if (c->flags & CLIENT_CLOSE_ASAP) { - ln = listSearchKey(server.clients_to_close,c); - serverAssert(ln != NULL); - listDelNode(server.clients_to_close,ln); - } + /* Remove the contribution that this client gave to our + * incrementally computed memory usage. */ + server.stat_clients_type_memory[c->client_cron_last_memory_type] -= + c->client_cron_last_memory_usage; /* Release other dynamically allocated client structure fields, * and finally release the client structure itself. */ @@ -949,20 +1217,43 @@ void freeClient(client *c) { * a context where calling freeClient() is not possible, because the client * should be valid for the continuation of the flow of the program. */ void freeClientAsync(client *c) { + /* We need to handle concurrent access to the server.clients_to_close list + * only in the freeClientAsync() function, since it's the only function that + * may access the list while Redis uses I/O threads. All the other accesses + * are in the context of the main thread while the other threads are + * idle. */ if (c->flags & CLIENT_CLOSE_ASAP || c->flags & CLIENT_LUA) return; c->flags |= CLIENT_CLOSE_ASAP; + if (server.io_threads_num == 1) { + /* no need to bother with locking if there's just one thread (the main thread) */ + listAddNodeTail(server.clients_to_close,c); + return; + } + static pthread_mutex_t async_free_queue_mutex = PTHREAD_MUTEX_INITIALIZER; + pthread_mutex_lock(&async_free_queue_mutex); listAddNodeTail(server.clients_to_close,c); + pthread_mutex_unlock(&async_free_queue_mutex); } -void freeClientsInAsyncFreeQueue(void) { - while (listLength(server.clients_to_close)) { - listNode *ln = listFirst(server.clients_to_close); +/* Free the clietns marked as CLOSE_ASAP, return the number of clients + * freed. */ +int freeClientsInAsyncFreeQueue(void) { + int freed = 0; + listIter li; + listNode *ln; + + listRewind(server.clients_to_close,&li); + while ((ln = listNext(&li)) != NULL) { client *c = listNodeValue(ln); + if (c->flags & CLIENT_PROTECTED) continue; + c->flags &= ~CLIENT_CLOSE_ASAP; freeClient(c); listDelNode(server.clients_to_close,ln); + freed++; } + return freed; } /* Return a client by ID, or NULL if the client ID is not in the set @@ -975,15 +1266,21 @@ client *lookupClientByID(uint64_t id) { } /* Write data in output buffers to client. Return C_OK if the client - * is still valid after the call, C_ERR if it was freed. */ -int writeToClient(int fd, client *c, int handler_installed) { + * is still valid after the call, C_ERR if it was freed because of some + * error. If handler_installed is set, it will attempt to clear the + * write event. + * + * This function is called by threads, but always with handler_installed + * set to 0. So when handler_installed is set to 0 the function must be + * thread safe. */ +int writeToClient(client *c, int handler_installed) { ssize_t nwritten = 0, totwritten = 0; size_t objlen; clientReplyBlock *o; while(clientHasPendingReplies(c)) { if (c->bufpos > 0) { - nwritten = write(fd,c->buf+c->sentlen,c->bufpos-c->sentlen); + nwritten = connWrite(c->conn,c->buf+c->sentlen,c->bufpos-c->sentlen); if (nwritten <= 0) break; c->sentlen += nwritten; totwritten += nwritten; @@ -1004,7 +1301,7 @@ int writeToClient(int fd, client *c, int handler_installed) { continue; } - nwritten = write(fd, o->buf + c->sentlen, objlen - c->sentlen); + nwritten = connWrite(c->conn, o->buf + c->sentlen, objlen - c->sentlen); if (nwritten <= 0) break; c->sentlen += nwritten; totwritten += nwritten; @@ -1030,8 +1327,8 @@ int writeToClient(int fd, client *c, int handler_installed) { * just deliver as much data as it is possible to deliver. * * Moreover, we also send as much as possible if the client is - * a slave (otherwise, on high-speed traffic, the replication - * buffer will grow indefinitely) */ + * a slave or a monitor (otherwise, on high-speed traffic, the + * replication/output buffer will grow indefinitely) */ if (totwritten > NET_MAX_WRITES_PER_EVENT && (server.maxmemory == 0 || zmalloc_used_memory() < server.maxmemory) && @@ -1039,12 +1336,12 @@ int writeToClient(int fd, client *c, int handler_installed) { } server.stat_net_output_bytes += totwritten; if (nwritten == -1) { - if (errno == EAGAIN) { + if (connGetState(c->conn) == CONN_STATE_CONNECTED) { nwritten = 0; } else { serverLog(LL_VERBOSE, - "Error writing to client: %s", strerror(errno)); - freeClient(c); + "Error writing to client: %s", connGetLastError(c->conn)); + freeClientAsync(c); return C_ERR; } } @@ -1057,11 +1354,15 @@ int writeToClient(int fd, client *c, int handler_installed) { } if (!clientHasPendingReplies(c)) { c->sentlen = 0; - if (handler_installed) aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); + /* Note that writeToClient() is called in a threaded way, but + * adDeleteFileEvent() is not thread safe: however writeToClient() + * is always called with handler_installed set to 0 from threads + * so we are fine. */ + if (handler_installed) connSetWriteHandler(c->conn, NULL); /* Close connection after entire reply has been sent. */ if (c->flags & CLIENT_CLOSE_AFTER_REPLY) { - freeClient(c); + freeClientAsync(c); return C_ERR; } } @@ -1069,10 +1370,9 @@ int writeToClient(int fd, client *c, int handler_installed) { } /* Write event handler. Just send data to the client. */ -void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) { - UNUSED(el); - UNUSED(mask); - writeToClient(fd,privdata,1); +void sendReplyToClient(connection *conn) { + client *c = connGetPrivateData(conn); + writeToClient(c,1); } /* This function is called just before entering the event loop, in the hope @@ -1095,26 +1395,24 @@ int handleClientsWithPendingWrites(void) { if (c->flags & CLIENT_PROTECTED) continue; /* Try to write buffers to the client socket. */ - if (writeToClient(c->fd,c,0) == C_ERR) continue; + if (writeToClient(c,0) == C_ERR) continue; /* If after the synchronous writes above we still have data to * output to the client, we need to install the writable handler. */ if (clientHasPendingReplies(c)) { - int ae_flags = AE_WRITABLE; + int ae_barrier = 0; /* For the fsync=always policy, we want that a given FD is never * served for reading and writing in the same event loop iteration, * so that in the middle of receiving the query, and serving it * to the client, we'll call beforeSleep() that will do the - * actual fsync of AOF to disk. AE_BARRIER ensures that. */ + * actual fsync of AOF to disk. the write barrier ensures that. */ if (server.aof_state == AOF_ON && server.aof_fsync == AOF_FSYNC_ALWAYS) { - ae_flags |= AE_BARRIER; + ae_barrier = 1; } - if (aeCreateFileEvent(server.el, c->fd, ae_flags, - sendReplyToClient, c) == AE_ERR) - { - freeClientAsync(c); + if (connSetWriteHandlerWithBarrier(c->conn, sendReplyToClient, ae_barrier) == C_ERR) { + freeClientAsync(c); } } } @@ -1135,6 +1433,12 @@ void resetClient(client *c) { if (!(c->flags & CLIENT_MULTI) && prevcmd != askingCommand) c->flags &= ~CLIENT_ASKING; + /* We do the same for the CACHING command as well. It also affects + * the next command or transaction executed, in a way very similar + * to ASKING. */ + if (!(c->flags & CLIENT_MULTI) && prevcmd != clientCommand) + c->flags &= ~CLIENT_TRACKING_CACHING; + /* Remove the CLIENT_REPLY_SKIP flag if any so that the reply * to the next command will be sent, but set the flag if the command * we just processed was "CLIENT REPLY SKIP". */ @@ -1160,15 +1464,15 @@ void resetClient(client *c) { * path, it is not really released, but only marked for later release. */ void protectClient(client *c) { c->flags |= CLIENT_PROTECTED; - aeDeleteFileEvent(server.el,c->fd,AE_READABLE); - aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); + connSetReadHandler(c->conn,NULL); + connSetWriteHandler(c->conn,NULL); } /* This will undo the client protection done by protectClient() */ void unprotectClient(client *c) { if (c->flags & CLIENT_PROTECTED) { c->flags &= ~CLIENT_PROTECTED; - aeCreateFileEvent(server.el,c->fd,AE_READABLE,readQueryFromClient,c); + connSetReadHandler(c->conn,readQueryFromClient); if (clientHasPendingReplies(c)) clientInstallWriteHandler(c); } } @@ -1216,9 +1520,22 @@ int processInlineBuffer(client *c) { /* Newline from slaves can be used to refresh the last ACK time. * This is useful for a slave to ping back while loading a big * RDB file. */ - if (querylen == 0 && c->flags & CLIENT_SLAVE) + if (querylen == 0 && getClientType(c) == CLIENT_TYPE_SLAVE) c->repl_ack_time = server.unixtime; + /* Masters should never send us inline protocol to run actual + * commands. If this happens, it is likely due to a bug in Redis where + * we got some desynchronization in the protocol, for example + * beause of a PSYNC gone bad. + * + * However the is an exception: masters may send us just a newline + * to keep the connection active. */ + if (querylen != 0 && c->flags & CLIENT_MASTER) { + serverLog(LL_WARNING,"WARNING: Receiving inline protocol from master, master stream corruption? Closing the master connection and discarding the cached master."); + setProtocolError("Master using the inline protocol. Desync?",c); + return C_ERR; + } + /* Move querybuffer position to the next query in the buffer. */ c->qb_pos += querylen+linefeed_chars; @@ -1230,22 +1547,19 @@ int processInlineBuffer(client *c) { /* Create redis objects for all arguments. */ for (c->argc = 0, j = 0; j < argc; j++) { - if (sdslen(argv[j])) { - c->argv[c->argc] = createObject(OBJ_STRING,argv[j]); - c->argc++; - } else { - sdsfree(argv[j]); - } + c->argv[c->argc] = createObject(OBJ_STRING,argv[j]); + c->argc++; } zfree(argv); return C_OK; } /* Helper function. Record protocol erro details in server log, - * and set the client as CLIENT_CLOSE_AFTER_REPLY. */ + * and set the client as CLIENT_CLOSE_AFTER_REPLY and + * CLIENT_PROTOCOL_ERROR. */ #define PROTO_DUMP_LEN 128 static void setProtocolError(const char *errstr, client *c) { - if (server.verbosity <= LL_VERBOSE) { + if (server.verbosity <= LL_VERBOSE || c->flags & CLIENT_MASTER) { sds client = catClientInfoString(sdsempty(),c); /* Sample some protocol to given an idea about what was inside. */ @@ -1264,11 +1578,13 @@ static void setProtocolError(const char *errstr, client *c) { } /* Log all the client and protocol info. */ - serverLog(LL_VERBOSE, + int loglevel = (c->flags & CLIENT_MASTER) ? LL_WARNING : + LL_VERBOSE; + serverLog(loglevel, "Protocol error (%s) from client: %s. %s", errstr, client, buf); sdsfree(client); } - c->flags |= CLIENT_CLOSE_AFTER_REPLY; + c->flags |= (CLIENT_CLOSE_AFTER_REPLY|CLIENT_PROTOCOL_ERROR); } /* Process the query buffer for client 'c', setting up the client argument @@ -1417,13 +1733,71 @@ int processMultibulkBuffer(client *c) { return C_ERR; } +/* Perform necessary tasks after a command was executed: + * + * 1. The client is reset unless there are reasons to avoid doing it. + * 2. In the case of master clients, the replication offset is updated. + * 3. Propagate commands we got from our master to replicas down the line. */ +void commandProcessed(client *c) { + long long prev_offset = c->reploff; + if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) { + /* Update the applied replication offset of our master. */ + c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos; + } + + /* Don't reset the client structure for clients blocked in a + * module blocking command, so that the reply callback will + * still be able to access the client argv and argc field. + * The client will be reset in unblockClientFromModule(). */ + if (!(c->flags & CLIENT_BLOCKED) || + c->btype != BLOCKED_MODULE) + { + resetClient(c); + } + + /* If the client is a master we need to compute the difference + * between the applied offset before and after processing the buffer, + * to understand how much of the replication stream was actually + * applied to the master state: this quantity, and its corresponding + * part of the replication stream, will be propagated to the + * sub-replicas and to the replication backlog. */ + if (c->flags & CLIENT_MASTER) { + long long applied = c->reploff - prev_offset; + if (applied) { + replicationFeedSlavesFromMasterStream(server.slaves, + c->pending_querybuf, applied); + sdsrange(c->pending_querybuf,applied,-1); + } + } +} + +/* This function calls processCommand(), but also performs a few sub tasks + * for the client that are useful in that context: + * + * 1. It sets the current client to the client 'c'. + * 2. calls commandProcessed() if the command was handled. + * + * The function returns C_ERR in case the client was freed as a side effect + * of processing the command, otherwise C_OK is returned. */ +int processCommandAndResetClient(client *c) { + int deadclient = 0; + server.current_client = c; + if (processCommand(c) == C_OK) { + commandProcessed(c); + } + if (server.current_client == NULL) deadclient = 1; + server.current_client = NULL; + /* freeMemoryIfNeeded may flush slave output buffers. This may + * result into a slave, that may be the active client, to be + * freed. */ + return deadclient ? C_ERR : C_OK; +} + /* This function is called every time, in the client structure 'c', there is * more query buffer to process, because we read more data from the socket * or because a client was blocked and later reactivated, so there could be * pending query buffer, already representing a full command, to process. */ void processInputBuffer(client *c) { - server.current_client = c; - /* Keep processing while there is something in the input buffer */ while(c->qb_pos < sdslen(c->querybuf)) { /* Return if clients are paused. */ @@ -1432,6 +1806,10 @@ void processInputBuffer(client *c) { /* Immediately abort if the client is in the middle of something. */ if (c->flags & CLIENT_BLOCKED) break; + /* Don't process more buffers from clients that have already pending + * commands to execute in c->argv. */ + if (c->flags & CLIENT_PENDING_COMMAND) break; + /* Don't process input from the master while there is a busy script * condition on the slave. We want just to accumulate the replication * stream (instead of replying -BUSY like we do with other clients) and @@ -1456,6 +1834,17 @@ void processInputBuffer(client *c) { if (c->reqtype == PROTO_REQ_INLINE) { if (processInlineBuffer(c) != C_OK) break; + /* If the Gopher mode and we got zero or one argument, process + * the request in Gopher mode. */ + if (server.gopher_enabled && + ((c->argc == 1 && ((char*)(c->argv[0]->ptr))[0] == '/') || + c->argc == 0)) + { + processGopherRequest(c); + resetClient(c); + c->flags |= CLIENT_CLOSE_AFTER_REPLY; + break; + } } else if (c->reqtype == PROTO_REQ_MULTIBULK) { if (processMultibulkBuffer(c) != C_OK) break; } else { @@ -1466,61 +1855,39 @@ void processInputBuffer(client *c) { if (c->argc == 0) { resetClient(c); } else { - /* Only reset the client when the command was executed. */ - if (processCommand(c) == C_OK) { - if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) { - /* Update the applied replication offset of our master. */ - c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos; - } + /* If we are in the context of an I/O thread, we can't really + * execute the command here. All we can do is to flag the client + * as one that needs to process the command. */ + if (c->flags & CLIENT_PENDING_READ) { + c->flags |= CLIENT_PENDING_COMMAND; + break; + } - /* Don't reset the client structure for clients blocked in a - * module blocking command, so that the reply callback will - * still be able to access the client argv and argc field. - * The client will be reset in unblockClientFromModule(). */ - if (!(c->flags & CLIENT_BLOCKED) || c->btype != BLOCKED_MODULE) - resetClient(c); + /* We are finally ready to execute the command. */ + if (processCommandAndResetClient(c) == C_ERR) { + /* If the client is no longer valid, we avoid exiting this + * loop and trimming the client buffer later. So we return + * ASAP in that case. */ + return; } - /* freeMemoryIfNeeded may flush slave output buffers. This may - * result into a slave, that may be the active client, to be - * freed. */ - if (server.current_client == NULL) break; } } /* Trim to pos */ - if (server.current_client != NULL && c->qb_pos) { + if (c->qb_pos) { sdsrange(c->querybuf,c->qb_pos,-1); c->qb_pos = 0; } - - server.current_client = NULL; -} - -/* This is a wrapper for processInputBuffer that also cares about handling - * the replication forwarding to the sub-slaves, in case the client 'c' - * is flagged as master. Usually you want to call this instead of the - * raw processInputBuffer(). */ -void processInputBufferAndReplicate(client *c) { - if (!(c->flags & CLIENT_MASTER)) { - processInputBuffer(c); - } else { - size_t prev_offset = c->reploff; - processInputBuffer(c); - size_t applied = c->reploff - prev_offset; - if (applied) { - replicationFeedSlavesFromMasterStream(server.slaves, - c->pending_querybuf, applied); - sdsrange(c->pending_querybuf,applied,-1); - } - } } -void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { - client *c = (client*) privdata; +void readQueryFromClient(connection *conn) { + client *c = connGetPrivateData(conn); int nread, readlen; size_t qblen; - UNUSED(el); - UNUSED(mask); + + /* Check if we want to read from the client later when exiting from + * the event loop. This is the case if threaded I/O is enabled. */ + if (postponeClientRead(c)) return; readlen = PROTO_IOBUF_LEN; /* If this is a multi bulk request, and we are processing a bulk reply @@ -1542,18 +1909,18 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { qblen = sdslen(c->querybuf); if (c->querybuf_peak < qblen) c->querybuf_peak = qblen; c->querybuf = sdsMakeRoomFor(c->querybuf, readlen); - nread = read(fd, c->querybuf+qblen, readlen); + nread = connRead(c->conn, c->querybuf+qblen, readlen); if (nread == -1) { - if (errno == EAGAIN) { + if (connGetState(conn) == CONN_STATE_CONNECTED) { return; } else { - serverLog(LL_VERBOSE, "Reading from client: %s",strerror(errno)); - freeClient(c); + serverLog(LL_VERBOSE, "Reading from client: %s",connGetLastError(c->conn)); + freeClientAsync(c); return; } } else if (nread == 0) { serverLog(LL_VERBOSE, "Client closed connection"); - freeClient(c); + freeClientAsync(c); return; } else if (c->flags & CLIENT_MASTER) { /* Append the query buffer to the pending (not applied) buffer @@ -1574,17 +1941,13 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { serverLog(LL_WARNING,"Closing client that reached max query buffer length: %s (qbuf initial bytes: %s)", ci, bytes); sdsfree(ci); sdsfree(bytes); - freeClient(c); + freeClientAsync(c); return; } - /* Time to process the buffer. If the client is a master we need to - * compute the difference between the applied offset before and after - * processing the buffer, to understand how much of the replication stream - * was actually applied to the master state: this quantity, and its - * corresponding part of the replication stream, will be propagated to - * the sub-slaves and to the replication backlog. */ - processInputBufferAndReplicate(c); + /* There is more data in the client input buffer, continue parsing it + * in case to check if there is a full command to execute. */ + processInputBuffer(c); } void getClientsMaxBuffers(unsigned long *longest_output_list, @@ -1623,7 +1986,7 @@ void genClientPeerId(client *client, char *peerid, snprintf(peerid,peerid_len,"%s:0",server.unixsocket); } else { /* TCP client. */ - anetFormatPeer(client->fd,peerid,peerid_len); + connFormatPeer(client->conn,peerid,peerid_len); } } @@ -1644,8 +2007,7 @@ char *getClientPeerId(client *c) { /* Concatenate a string representing the state of a client in an human * readable format, into the sds string 's'. */ sds catClientInfoString(sds s, client *client) { - char flags[16], events[3], *p; - int emask; + char flags[16], events[3], conninfo[CONN_INFO_LEN], *p; p = flags; if (client->flags & CLIENT_SLAVE) { @@ -1658,6 +2020,8 @@ sds catClientInfoString(sds s, client *client) { if (client->flags & CLIENT_PUBSUB) *p++ = 'P'; if (client->flags & CLIENT_MULTI) *p++ = 'x'; if (client->flags & CLIENT_BLOCKED) *p++ = 'b'; + if (client->flags & CLIENT_TRACKING) *p++ = 't'; + if (client->flags & CLIENT_TRACKING_BROKEN_REDIR) *p++ = 'R'; if (client->flags & CLIENT_DIRTY_CAS) *p++ = 'd'; if (client->flags & CLIENT_CLOSE_AFTER_REPLY) *p++ = 'c'; if (client->flags & CLIENT_UNBLOCKED) *p++ = 'u'; @@ -1667,16 +2031,17 @@ sds catClientInfoString(sds s, client *client) { if (p == flags) *p++ = 'N'; *p++ = '\0'; - emask = client->fd == -1 ? 0 : aeGetFileEvents(server.el,client->fd); p = events; - if (emask & AE_READABLE) *p++ = 'r'; - if (emask & AE_WRITABLE) *p++ = 'w'; + if (client->conn) { + if (connHasReadHandler(client->conn)) *p++ = 'r'; + if (connHasWriteHandler(client->conn)) *p++ = 'w'; + } *p = '\0'; return sdscatfmt(s, - "id=%U addr=%s fd=%i name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U obl=%U oll=%U omem=%U events=%s cmd=%s", + "id=%U addr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U obl=%U oll=%U omem=%U events=%s cmd=%s user=%s", (unsigned long long) client->id, getClientPeerId(client), - client->fd, + connGetInfo(client->conn, conninfo, sizeof(conninfo)), client->name ? (char*)client->name->ptr : "", (long long)(server.unixtime - client->ctime), (long long)(server.unixtime - client->lastinteraction), @@ -1691,7 +2056,8 @@ sds catClientInfoString(sds s, client *client) { (unsigned long long) listLength(client->reply), (unsigned long long) getClientOutputBufferMemoryUsage(client), events, - client->lastcmd ? client->lastcmd->name : "NULL"); + client->lastcmd ? client->lastcmd->name : "NULL", + client->user ? client->user->name : "(superuser)"); } sds getAllClientsInfoString(int type) { @@ -1710,26 +2076,66 @@ sds getAllClientsInfoString(int type) { return o; } +/* This function implements CLIENT SETNAME, including replying to the + * user with an error if the charset is wrong (in that case C_ERR is + * returned). If the function succeeeded C_OK is returned, and it's up + * to the caller to send a reply if needed. + * + * Setting an empty string as name has the effect of unsetting the + * currently set name: the client will remain unnamed. + * + * This function is also used to implement the HELLO SETNAME option. */ +int clientSetNameOrReply(client *c, robj *name) { + int len = sdslen(name->ptr); + char *p = name->ptr; + + /* Setting the client name to an empty string actually removes + * the current name. */ + if (len == 0) { + if (c->name) decrRefCount(c->name); + c->name = NULL; + return C_OK; + } + + /* Otherwise check if the charset is ok. We need to do this otherwise + * CLIENT LIST format will break. You should always be able to + * split by space to get the different fields. */ + for (int j = 0; j < len; j++) { + if (p[j] < '!' || p[j] > '~') { /* ASCII is assumed. */ + addReplyError(c, + "Client names cannot contain spaces, " + "newlines or special characters."); + return C_ERR; + } + } + if (c->name) decrRefCount(c->name); + c->name = name; + incrRefCount(name); + return C_OK; +} + void clientCommand(client *c) { listNode *ln; listIter li; - client *client; if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { const char *help[] = { -"id -- Return the ID of the current connection.", -"getname -- Return the name of the current connection.", -"kill -- Kill connection made from .", -"kill